From 7820e2f64bdf09dcfe0e797af2e0885369ca19ae Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Wed, 7 Mar 2018 09:48:33 +0200 Subject: [PATCH] Initial commit Signed-off-by: Jonathan Dieter --- .gitignore | 1 + include/meson.build | 1 + include/zck.h | 100 ++++ meson.build | 33 ++ meson_options.txt | 1 + src/lib/comp/comp.c | 191 +++++++ src/lib/comp/meson.build | 5 + src/lib/comp/nocomp/meson.build | 1 + src/lib/comp/nocomp/nocomp.c | 77 +++ src/lib/comp/nocomp/nocomp.h | 6 + src/lib/comp/zstd/meson.build | 1 + src/lib/comp/zstd/zstd.c | 167 ++++++ src/lib/comp/zstd/zstd.h | 6 + src/lib/dl/README.md | 98 ++++ src/lib/dl/dl.c | 215 ++++++++ src/lib/dl/meson.build | 1 + src/lib/dl/multipart_parser.c | 306 ++++++++++ src/lib/dl/multipart_parser.h | 48 ++ src/lib/dl/range.c | 229 ++++++++ src/lib/hash/hash.c | 147 +++++ src/lib/hash/meson.build | 3 + src/lib/hash/sha1/meson.build | 1 + src/lib/hash/sha1/sha1.c | 157 ++++++ src/lib/hash/sha1/sha1.h | 62 +++ src/lib/hash/sha2/meson.build | 1 + src/lib/hash/sha2/sha2.c | 949 ++++++++++++++++++++++++++++++++ src/lib/hash/sha2/sha2.h | 108 ++++ src/lib/header.c | 140 +++++ src/lib/index/index_common.c | 64 +++ src/lib/index/index_create.c | 207 +++++++ src/lib/index/index_read.c | 146 +++++ src/lib/index/meson.build | 1 + src/lib/io.c | 74 +++ src/lib/log.c | 47 ++ src/lib/meson.build | 13 + src/lib/zck.c | 338 ++++++++++++ src/lib/zck_private.h | 122 ++++ src/memmem.c | 64 +++ src/memmem.h | 4 + src/meson.build | 6 + src/unzck.c | 80 +++ src/zck.c | 212 +++++++ src/zck_delta_size.c | 108 ++++ src/zck_dl.c | 146 +++++ src/zck_read_header.c | 79 +++ 45 files changed, 4766 insertions(+) create mode 100644 .gitignore create mode 100644 include/meson.build create mode 100644 include/zck.h create mode 100644 meson.build create mode 100644 meson_options.txt create mode 100644 src/lib/comp/comp.c create mode 100644 src/lib/comp/meson.build create mode 100644 src/lib/comp/nocomp/meson.build create mode 100644 src/lib/comp/nocomp/nocomp.c create mode 100644 src/lib/comp/nocomp/nocomp.h create mode 100644 src/lib/comp/zstd/meson.build create mode 100644 src/lib/comp/zstd/zstd.c create mode 100644 src/lib/comp/zstd/zstd.h create mode 100644 src/lib/dl/README.md create mode 100644 src/lib/dl/dl.c create mode 100644 src/lib/dl/meson.build create mode 100644 src/lib/dl/multipart_parser.c create mode 100644 src/lib/dl/multipart_parser.h create mode 100644 src/lib/dl/range.c create mode 100644 src/lib/hash/hash.c create mode 100644 src/lib/hash/meson.build create mode 100644 src/lib/hash/sha1/meson.build create mode 100644 src/lib/hash/sha1/sha1.c create mode 100644 src/lib/hash/sha1/sha1.h create mode 100644 src/lib/hash/sha2/meson.build create mode 100644 src/lib/hash/sha2/sha2.c create mode 100644 src/lib/hash/sha2/sha2.h create mode 100644 src/lib/header.c create mode 100644 src/lib/index/index_common.c create mode 100644 src/lib/index/index_create.c create mode 100644 src/lib/index/index_read.c create mode 100644 src/lib/index/meson.build create mode 100644 src/lib/io.c create mode 100644 src/lib/log.c create mode 100644 src/lib/meson.build create mode 100644 src/lib/zck.c create mode 100644 src/lib/zck_private.h create mode 100644 src/memmem.c create mode 100644 src/memmem.h create mode 100644 src/meson.build create mode 100644 src/unzck.c create mode 100644 src/zck.c create mode 100644 src/zck_delta_size.c create mode 100644 src/zck_dl.c create mode 100644 src/zck_read_header.c diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..378eac2 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +build diff --git a/include/meson.build b/include/meson.build new file mode 100644 index 0000000..acc57cd --- /dev/null +++ b/include/meson.build @@ -0,0 +1 @@ +install_headers('zck.h') diff --git a/include/zck.h b/include/zck.h new file mode 100644 index 0000000..8845308 --- /dev/null +++ b/include/zck.h @@ -0,0 +1,100 @@ +#ifndef ZCK_H +#define ZCK_H + +#define ZCK_VERSION "0.0.1" +#define ZCK_VER_MAJOR 0 +#define ZCK_VER_MINOR 0 +#define ZCK_VER_REVISION 1 +#define ZCK_VER_SUBREVISION 0 + +#define ZCK_COMP_NONE 0 +#define ZCK_COMP_ZSTD 2 + +#define ZCK_HASH_SHA1 0 +#define ZCK_HASH_SHA256 1 + +#define ZCK_COMMON_DICT 1 +#define ZCK_COMMON_DICT_SIZE 2 + +#define ZCK_ZCK_COMP_LEVEL 20 + +#define True 1 +#define False 0 + +typedef enum log_type { ZCK_LOG_DEBUG, ZCK_LOG_INFO, ZCK_LOG_WARNING, + ZCK_LOG_ERROR } log_type; + + +typedef struct zckRange { + uint64_t start; + uint64_t end; + struct zckRange *next; + struct zckRange *prev; +} zckRange; + +typedef struct zckRangeInfo { + unsigned int count; + unsigned int segments; + unsigned int max_ranges; + zckRange *first; +} zckRangeInfo; + +typedef struct zckIndex { + char *digest; + uint64_t start; + size_t length; + struct zckIndex *next; +} zckIndex; + +typedef struct zckHashType zckHashType; + +typedef struct zckIndexInfo { + uint64_t count; + size_t length; + zckHashType *hash_type; + zckIndex *first; +} zckIndexInfo; + +typedef struct zckCtx zckCtx; +typedef struct zckDL zckDL; + +zckCtx *zck_create(); +void zck_free(zckCtx *zck); +int zck_init_write (zckCtx *zck, int dst_fd); +int zck_set_compression_type(zckCtx *zck, uint8_t comp_type); +int zck_set_comp_parameter(zckCtx *zck, int option, void *value); +int zck_comp_init(zckCtx *zck); +int zck_comp_close(zckCtx *zck); +int zck_compress(zckCtx *zck, const char *src, const size_t src_size); +int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, + char **dst, size_t *dst_size); +int zck_write_file(zckCtx *zck); +int zck_read_header(zckCtx *zck, int src_fd); +uint64_t zck_get_index_count(zckCtx *zck); +zckIndexInfo *zck_get_index(zckCtx *zck); +int zck_decompress_to_file (zckCtx *zck, int src_fd, int dst_fd); +int zck_set_full_hash_type(zckCtx *zck, uint8_t hash_type); +int zck_set_chunk_hash_type(zckCtx *zck, uint8_t hash_type); +char *zck_get_index_digest(zckCtx *zck); +char *zck_get_full_digest(zckCtx *zck); +int zck_get_full_digest_size(zckCtx *zck); +int zck_get_chunk_digest_size(zckCtx *zck); +uint8_t zck_get_full_hash_type(zckCtx *zck); +uint8_t zck_get_chunk_hash_type(zckCtx *zck); +const char *zck_hash_name_from_type(uint8_t hash_type); +const char *zck_comp_name_from_type(uint8_t comp_type); +int zck_range_calc_segments(zckRangeInfo *info, unsigned int max_ranges); +int zck_range_get_need_dl(zckRangeInfo *info, zckCtx *zck_src, zckCtx *zck_tgt); +int zck_range_get_array(zckRangeInfo *info, char **ra); +void zck_range_close(zckRangeInfo *info); +void zck_set_log_level(log_type ll); +void zck_dl_global_init(); +void zck_dl_global_cleanup(); +zckDL *zck_dl_init(); +void zck_dl_free(zckDL *dl); +int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url); +size_t zck_dl_get_bytes_downloaded(zckDL *dl); +size_t zck_dl_get_bytes_uploaded(zckDL *dl); +char *zck_dl_get_range(unsigned int start, unsigned int end); +#endif + diff --git a/meson.build b/meson.build new file mode 100644 index 0000000..a670ac6 --- /dev/null +++ b/meson.build @@ -0,0 +1,33 @@ +project('zck', 'c') + +lib_version = '0.0.1' +so_version = '1' + +use_zstd = get_option('with-zstd') +zstd_dep = [] +if use_zstd == 'yes' + zstd_required = true +else + zstd_required = false +endif +if use_zstd == 'auto' or use_zstd == 'yes' + zstd_dep = dependency('libzstd', required:zstd_required) + if zstd_dep.found() + add_global_arguments('-DZCHUNK_ZSTD', language : 'c') + use_zstd = 'yes' + endif +endif +curl_dep = dependency('libcurl', required:true) + +inc = include_directories('include') + +subdir('include') +subdir('src') +subdir('test') + +pkg_mod = import('pkgconfig') +pkg_mod.generate(libraries: zcklib, + version: lib_version, + name: 'libzck', + filebase: 'zck', + description: 'A library for generating easy-to-delta files.') diff --git a/meson_options.txt b/meson_options.txt new file mode 100644 index 0000000..6dc1a6d --- /dev/null +++ b/meson_options.txt @@ -0,0 +1 @@ +option('with-zstd', type : 'combo', choices : ['yes', 'no', 'auto'], value : 'auto') diff --git a/src/lib/comp/comp.c b/src/lib/comp/comp.c new file mode 100644 index 0000000..153f2f8 --- /dev/null +++ b/src/lib/comp/comp.c @@ -0,0 +1,191 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "zck_private.h" +#include "comp/nocomp/nocomp.h" +#ifdef ZCHUNK_ZSTD +#include "comp/zstd/zstd.h" +#endif + +#define BLK_SIZE 32768 + +static char unknown[] = "Unknown(\0\0\0\0\0"; + +const static char *COMP_NAME[] = { + "no", + "zstd" +}; + +int zck_comp_init(zckCtx *zck) { + zckComp *comp = &(zck->comp); + char *dst = NULL; + size_t dst_size = 0; + + if(zck->comp.started) { + zck_log(ZCK_LOG_ERROR, "Compression already initialized\n"); + return False; + } + if((zck->comp.dict && zck->comp.dict_size == 0) || + (zck->comp.dict == NULL && zck->comp.dict_size > 0)) { + zck_log(ZCK_LOG_ERROR, "Invalid dictionary configuration\n"); + return False; + } + if(!zck->comp.init(&(zck->comp))) + return False; + + if(zck->comp.dict && zck->temp_fd) { + if(!zck->comp.compress(comp, zck->comp.dict, zck->comp.dict_size, &dst, + &dst_size, 0)) + return False; + + if(!zck_write(zck->temp_fd, dst, dst_size)) { + free(dst); + return False; + } + zck_index_add_chunk(zck, dst, dst_size); + free(dst); + } + zck->comp.dict = NULL; + zck->comp.dict_size = 0; + zck->comp.started = True; + return True; +} + +int zck_compress(zckCtx *zck, const char *src, const size_t src_size) { + zckComp *comp = &(zck->comp); + char *dst; + size_t dst_size; + + if(!zck->comp.started) { + zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n"); + return False; + } + + if(src_size == 0) + return True; + + if(!zck->comp.compress(comp, src, src_size, &dst, &dst_size, 1)) + return False; + + if(!zck_write(zck->temp_fd, dst, dst_size)) { + free(dst); + return False; + } + zck_index_add_chunk(zck, dst, dst_size); + free(dst); + return True; +} + +int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, char **dst, size_t *dst_size) { + zckComp *comp = &(zck->comp); + *dst = NULL; + *dst_size = 0; + + if(!zck->comp.started) { + zck_log(ZCK_LOG_ERROR, "Compression hasn't been initialized yet\n"); + return False; + } + + if(src_size == 0) + return True; + + if(!zck->comp.decompress(comp, src, src_size, dst, dst_size, 1)) + return False; + + return True; +} + +int zck_comp_close(zckCtx *zck) { + if(zck->comp.cctx) { + zck->comp.started = 0; + return zck->comp.close(&(zck->comp)); + } + return True; +} + +int zck_set_compression_type(zckCtx *zck, uint8_t type) { + zckComp *comp = &(zck->comp); + + /* Cannot change compression type after compression has started */ + if(comp->started) { + zck_log(ZCK_LOG_ERROR, + "Unable to set compression type after initialization\n"); + return False; + } + + /* Set all values to 0 before setting compression type */ + memset(comp, 0, sizeof(zckComp)); + if(type == ZCK_COMP_NONE) { + return zck_nocomp_setup(comp); +#ifdef ZCHUNK_ZSTD + } else if(type == ZCK_COMP_ZSTD) { + return zck_zstd_setup(comp); +#endif + } else { + zck_log(ZCK_LOG_ERROR, "Unsupported compression type: %s\n", + zck_comp_name_from_type(type)); + return False; + } + zck_log(ZCK_LOG_DEBUG, "Setting compression to %s\n", + zck_comp_name_from_type(type)); + return True; +} + +int zck_set_comp_parameter(zckCtx *zck, int option, void *value) { + /* Cannot change compression parameters after compression has started */ + if(zck && zck->comp.started) { + zck_log(ZCK_LOG_ERROR, + "Unable to set compression parameters after initialization\n"); + return False; + } + if(option == ZCK_COMMON_DICT) { + zck->comp.dict = value; + }else if(option == ZCK_COMMON_DICT_SIZE) { + zck->comp.dict_size = *(size_t*)value; + }else { + if(zck && zck->comp.set_parameter) + return zck->comp.set_parameter(&(zck->comp), option, value); + + zck_log(ZCK_LOG_ERROR, "Unsupported compression parameter: %i\n", + option); + return False; + } + return True; +} + +const char *zck_comp_name_from_type(uint8_t comp_type) { + if(comp_type > 1) { + snprintf(unknown+8, 4, "%i)", comp_type); + return unknown; + } + return COMP_NAME[comp_type]; +} diff --git a/src/lib/comp/meson.build b/src/lib/comp/meson.build new file mode 100644 index 0000000..0690aa0 --- /dev/null +++ b/src/lib/comp/meson.build @@ -0,0 +1,5 @@ +sources += ['comp/comp.c'] +if use_zstd == 'yes' + subdir('zstd') +endif +subdir('nocomp') diff --git a/src/lib/comp/nocomp/meson.build b/src/lib/comp/nocomp/meson.build new file mode 100644 index 0000000..0a5e5c3 --- /dev/null +++ b/src/lib/comp/nocomp/meson.build @@ -0,0 +1 @@ +sources += ['comp/nocomp/nocomp.c'] diff --git a/src/lib/comp/nocomp/nocomp.c b/src/lib/comp/nocomp/nocomp.c new file mode 100644 index 0000000..abeade7 --- /dev/null +++ b/src/lib/comp/nocomp/nocomp.c @@ -0,0 +1,77 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +static int zck_nocomp_init(zckComp *comp) { + return True; +} + +/* Nocomp just copies data for both compression and decompression */ +static int zck_nocomp_de_comp(zckComp *comp, const char *src, const size_t src_size, + char **dst, size_t *dst_size, int use_dict) { + *dst = zmalloc(src_size); + if(dst == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", src_size); + return False; + } + + memcpy(*dst, src, src_size); + *dst_size = src_size; + + return True; +} + +static int zck_nocomp_close(zckComp *comp) { + return True; +} + +/* Nocomp doesn't support any parameters, so return error if setting a parameter + * was attempted */ +static int zck_nocomp_set_parameter(zckComp *comp, int option, void *value) { + zck_log(ZCK_LOG_ERROR, "Invalid compression parameter for ZCK_COMP_NONE\n"); + return False; +} + +/* No default parameters to set when there's no compression */ +static int zck_nocomp_set_default_parameters(zckComp *comp) { + return True; +} + +int zck_nocomp_setup(zckComp *comp) { + comp->init = zck_nocomp_init; + comp->set_parameter = zck_nocomp_set_parameter; + comp->compress = zck_nocomp_de_comp; + comp->decompress = zck_nocomp_de_comp; + comp->close = zck_nocomp_close; + comp->type = ZCK_COMP_NONE; + return zck_nocomp_set_default_parameters(comp); +} diff --git a/src/lib/comp/nocomp/nocomp.h b/src/lib/comp/nocomp/nocomp.h new file mode 100644 index 0000000..327a300 --- /dev/null +++ b/src/lib/comp/nocomp/nocomp.h @@ -0,0 +1,6 @@ +#ifndef ZCHUNK_COMPRESSION_NOCOMP_H +#define ZCHUNK_COMPRESSION_NOCOMP_H + +int zck_nocomp_setup(zckComp *comp); + +#endif diff --git a/src/lib/comp/zstd/meson.build b/src/lib/comp/zstd/meson.build new file mode 100644 index 0000000..2bd8b5b --- /dev/null +++ b/src/lib/comp/zstd/meson.build @@ -0,0 +1 @@ +sources += ['comp/zstd/zstd.c'] diff --git a/src/lib/comp/zstd/zstd.c b/src/lib/comp/zstd/zstd.c new file mode 100644 index 0000000..fc7dd06 --- /dev/null +++ b/src/lib/comp/zstd/zstd.c @@ -0,0 +1,167 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +static int zck_zstd_init(zckComp *comp) { + comp->cctx = ZSTD_createCCtx(); + comp->dctx = ZSTD_createDCtx(); + if(comp->dict && comp->dict_size > 0) { + comp->cdict_ctx = ZSTD_createCDict(comp->dict, comp->dict_size, + comp->level); + if(comp->cdict_ctx == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to create zstd compression dict context\n"); + return False; + } + comp->ddict_ctx = ZSTD_createDDict(comp->dict, comp->dict_size); + if(comp->ddict_ctx == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to create zstd decompression dict context\n"); + return False; + } + } + return True; +} + +static int zck_zstd_compress(zckComp *comp, const char *src, + const size_t src_size, char **dst, + size_t *dst_size, int use_dict) { + size_t max_size = ZSTD_compressBound(src_size); + if(ZSTD_isError(max_size)) { + zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n", + ZSTD_getErrorName(max_size)); + return False; + } + + *dst = zmalloc(max_size); + if(dst == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", max_size); + return False; + } + + if(use_dict && comp->cdict_ctx) { + *dst_size = ZSTD_compress_usingCDict(comp->cctx, *dst, max_size, src, + src_size, comp->cdict_ctx); + } else { + *dst_size = ZSTD_compressCCtx(comp->cctx, *dst, max_size, src, src_size, + comp->level); + } + if(ZSTD_isError(*dst_size)) { + zck_log(ZCK_LOG_ERROR, "zstd compression error: %s\n", + ZSTD_getErrorName(*dst_size)); + return False; + } + return True; +} + +static int zck_zstd_decompress(zckComp *comp, const char *src, + const size_t src_size, char **dst, + size_t *dst_size, int use_dict) { + unsigned long long frame_size; + size_t retval; + + frame_size = ZSTD_getFrameContentSize(src, src_size); + if(frame_size == ZSTD_CONTENTSIZE_UNKNOWN || + frame_size == ZSTD_CONTENTSIZE_ERROR) { + zck_log(ZCK_LOG_ERROR, "Unknown content size\n"); + return False; + } + *dst_size = (size_t)frame_size; + *dst = zmalloc(*dst_size); + if(dst == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", *dst_size); + return False; + } + + if(use_dict && comp->ddict_ctx) { + retval = ZSTD_decompress_usingDDict(comp->dctx, *dst, *dst_size, src, + src_size, comp->ddict_ctx); + } else { + retval = ZSTD_decompressDCtx(comp->dctx, *dst, *dst_size, src, + src_size); + } + if(ZSTD_isError(retval)) { + free(*dst); + *dst = NULL; + zck_log(ZCK_LOG_ERROR, "zstd decompression error: %s\n", + ZSTD_getErrorName(retval)); + return False; + } + return True; +} + +static int zck_zstd_close(zckComp *comp) { + if(comp->cdict_ctx) { + ZSTD_freeCDict(comp->cdict_ctx); + comp->cdict_ctx = NULL; + } + if(comp->ddict_ctx) { + ZSTD_freeDDict(comp->ddict_ctx); + comp->ddict_ctx = NULL; + } + if(comp->cctx) { + ZSTD_freeCCtx(comp->cctx); + comp->cctx = NULL; + } + if(comp->dctx) { + ZSTD_freeDCtx(comp->dctx); + comp->dctx = NULL; + } + return True; +} + +static int zck_zstd_set_parameter(zckComp *comp, int option, void *value) { + if(option == ZCK_ZCK_COMP_LEVEL) { + if(*(int*)value >= 0 && *(int*)value <= ZSTD_maxCLevel()) { + comp->level = *(int*)value; + return True; + } + } + zck_log(ZCK_LOG_ERROR, "Invalid compression parameter for ZCK_COMP_ZSTD\n"); + return False; +} + +static int zck_zstd_set_default_parameters(zckComp *comp) { + /* Set default compression level to 16 */ + int level=16; + return zck_zstd_set_parameter(comp, ZCK_ZCK_COMP_LEVEL, &level); +} + +int zck_zstd_setup(zckComp *comp) { + comp->init = zck_zstd_init; + comp->set_parameter = zck_zstd_set_parameter; + comp->compress = zck_zstd_compress; + comp->decompress = zck_zstd_decompress; + comp->close = zck_zstd_close; + comp->type = ZCK_COMP_ZSTD; + return zck_zstd_set_default_parameters(comp); +} diff --git a/src/lib/comp/zstd/zstd.h b/src/lib/comp/zstd/zstd.h new file mode 100644 index 0000000..73b151d --- /dev/null +++ b/src/lib/comp/zstd/zstd.h @@ -0,0 +1,6 @@ +#ifndef ZCHUNK_COMPRESSION_ZSTD_H +#define ZCHUNK_COMPRESSION_ZSTD_H + +int zck_zstd_setup(zckComp *comp); + +#endif diff --git a/src/lib/dl/README.md b/src/lib/dl/README.md new file mode 100644 index 0000000..455e4da --- /dev/null +++ b/src/lib/dl/README.md @@ -0,0 +1,98 @@ +## Multipart form data parser + +### Features +* No dependencies +* Works with chunks of a data - no need to buffer the whole request +* Almost no internal buffering. Buffer size doesn't exceed the size of the boundary (~60-70 bytes) + +Tested as part of [Cosmonaut](https://github.com/iafonov/cosmonaut) HTTP server. + +Implementation based on [node-formidable](https://github.com/felixge/node-formidable) by [Felix Geisendörfer](https://github.com/felixge). + +Inspired by [http-parser](https://github.com/joyent/http-parser) by [Ryan Dahl](https://github.com/ry). + +### Usage (C) +This parser library works with several callbacks, which the user may set up at application initialization time. + +```c +multipart_parser_settings callbacks; + +memset(&callbacks, 0, sizeof(multipart_parser_settings)); + +callbacks.on_header_field = read_header_name; +callbacks.on_header_value = read_header_value; +``` + +These functions must match the signatures defined in the multipart-parser header file. For this simple example, we'll just use two of the available callbacks to print all headers the library finds in multipart messages. + +Returning a value other than 0 from the callbacks will abort message processing. + +```c +int read_header_name(multipart_parser* p, const char *at, size_t length) +{ + printf("%.*s: ", length, at); + return 0; +} + +int read_header_value(multipart_parser* p, const char *at, size_t length) +{ + printf("%.*s\n", length, at); + return 0; +} +``` + +When a message arrives, callers must parse the multipart boundary from the **Content-Type** header (see the [RFC](http://tools.ietf.org/html/rfc2387#section-5.1) for more information and examples), and then execute the parser. + +```c +multipart_parser* parser = multipart_parser_init(boundary, &callbacks); +multipart_parser_execute(parser, body, length); +multipart_parser_free(parser); +``` + +### Usage (C++) +In C++, when the callbacks are static member functions it may be helpful to pass the instantiated multipart consumer along as context. The following (abbreviated) class called `MultipartConsumer` shows how to pass `this` to callback functions in order to access non-static member data. + +```cpp +class MultipartConsumer +{ +public: + MultipartConsumer(const std::string& boundary) + { + memset(&m_callbacks, 0, sizeof(multipart_parser_settings)); + m_callbacks.on_header_field = ReadHeaderName; + m_callbacks.on_header_value = ReadHeaderValue; + + m_parser = multipart_parser_init(boundary.c_str(), &m_callbacks); + multipart_parser_set_data(m_parser, this); + } + + ~MultipartConsumer() + { + multipart_parser_free(m_parser); + } + + int CountHeaders(const std::string& body) + { + multipart_parser_execute(m_parser, body.c_str(), body.size()); + return m_headers; + } + +private: + static int ReadHeaderName(multipart_parser* p, const char *at, size_t length) + { + MultipartConsumer* me = (MultipartConsumer*)multipart_parser_get_data(p); + me->m_headers++; + } + + multipart_parser* m_parser; + multipart_parser_settings m_callbacks; + int m_headers; +}; +``` + +### Contributors +* [Daniel T. Wagner](http://www.danieltwagner.de/) +* [James McLaughlin](http://udp.github.com/) +* [Jay Miller](http://www.cryptofreak.org) + +© 2012 [Igor Afonov](http://iafonov.github.com) diff --git a/src/lib/dl/dl.c b/src/lib/dl/dl.c new file mode 100644 index 0000000..9e80ca2 --- /dev/null +++ b/src/lib/dl/dl.c @@ -0,0 +1,215 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "zck_private.h" + +#define VALIDATE(f) if(!f) { \ + zck_log(ZCK_LOG_ERROR, "zckDL not allocated\n"); \ + return False; \ + } +zckDL *zck_dl_init() { + zckDL *dl = zmalloc(sizeof(zckDL)); + if(!dl) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckDL)); + return NULL; + } + + dl->curl_ctx = curl_easy_init(); + if(!dl->curl_ctx) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(CURL)); + return NULL; + } + return dl; +} + +void zck_dl_free(zckDL *dl) { + if(!dl) + return; + curl_easy_cleanup(dl->curl_ctx); + free(dl); + dl = NULL; +} + +char *zck_dl_get_range_char(unsigned int start, unsigned int end) { + zckRange r = {0}; + zckRange *range = &r; + + r.start = start; + r.end = end; + char *range_header = zck_range_get_char(&range, 2); + return range_header; +} + +static size_t write_data(void *ptr, size_t size, size_t nmemb, void *stream) { + size_t wb = write(*(int*)stream, ptr, size*nmemb); + return wb; +} + +int zck_dl_range(zckDL *dl, char *url, int dst_fd, zckRangeInfo *info) { + if(info == NULL || info->first == NULL) { + zck_log(ZCK_LOG_ERROR, "Range not defined\n"); + return False; + } + if(info->segments == 0) + info->segments = 1; + + char **ra = calloc(sizeof(char*), info->segments); + if(!zck_range_get_array(info, ra)) { + free(ra); + return False; + } + CURLcode res; + + for(int i=0; isegments; i++) { + struct curl_slist *header = NULL; + double size; + header = curl_slist_append(header, ra[i]); + curl_easy_setopt(dl->curl_ctx, CURLOPT_URL, url); + curl_easy_setopt(dl->curl_ctx, CURLOPT_FOLLOWLOCATION, 1L); + curl_easy_setopt(dl->curl_ctx, CURLOPT_WRITEFUNCTION, write_data); + curl_easy_setopt(dl->curl_ctx, CURLOPT_WRITEDATA, &dst_fd); + curl_easy_setopt(dl->curl_ctx, CURLOPT_HTTPHEADER, header); + res = curl_easy_perform(dl->curl_ctx); + curl_slist_free_all(header); + free(ra[i]); + if(res != CURLE_OK) { + zck_log(ZCK_LOG_ERROR, "Download failed: %s\n", + curl_easy_strerror(res)); + return False; + } + long code; + curl_easy_getinfo (dl->curl_ctx, CURLINFO_RESPONSE_CODE, &code); + res = curl_easy_getinfo(dl->curl_ctx, CURLINFO_CONTENT_LENGTH_DOWNLOAD, + &size); + if(res != CURLE_OK) + zck_log(ZCK_LOG_WARNING, "Unable to get download size\n"); + else + dl->dl += (size_t)(size + 0.5); + if (code != 206 && code != 200) { + zck_log(ZCK_LOG_ERROR, "HTTP Error: %li when download %s\n", code, url); + return False; + } + } + free(ra); + return True; +} + +int zck_dl_bytes(zckDL *dl, char *url, int dst_fd, size_t bytes, size_t start, + size_t *buffer_len) { + if(start + bytes > *buffer_len) { + zck_log(ZCK_LOG_DEBUG, "Seeking to end of temporary file\n"); + if(lseek(dst_fd, 0, SEEK_END) == -1) { + zck_log(ZCK_LOG_ERROR, "Seek to end of temporary file failed: %s\n", + strerror(errno)); + return False; + } + zck_log(ZCK_LOG_DEBUG, "Downloading %lu bytes at position %lu\n", start+bytes-*buffer_len, *buffer_len); + zckRangeInfo info = {0}; + zck_range_add(&info, *buffer_len, start+bytes-1); + if(!zck_dl_range(dl, url, dst_fd, &info)) + return False; + zck_range_close(&info); + *buffer_len = start+bytes; + zck_log(ZCK_LOG_DEBUG, "Seeking to position %lu\n", start); + if(lseek(dst_fd, start, SEEK_SET) == -1) { + zck_log(ZCK_LOG_ERROR, + "Seek to byte %lu of temporary file failed: %s\n", start, + strerror(errno)); + return False; + } + } + return True; +} + +int zck_dl_get_header(zckCtx *zck, zckDL *dl, char *url) { + size_t buffer_len = 0; + size_t start = 0; + int temp_fd = zck_get_tmp_fd(); + + if(!zck_dl_bytes(dl, url, temp_fd, 100, start, &buffer_len)) + return False; + if(!zck_read_initial(zck, temp_fd)) + return False; + start += 6; + if(!zck_dl_bytes(dl, url, temp_fd, zck->hash_type.digest_size+9, start, + &buffer_len)) + return False; + if(!zck_read_index_hash(zck, temp_fd)) + return False; + start += zck->hash_type.digest_size; + + char *digest = zck_get_index_digest(zck); + zck_log(ZCK_LOG_DEBUG, "Index hash: (%s)", zck_hash_name_from_type(zck_get_full_hash_type(zck))); + for(int i=0; icomp_index_size); + if(!zck_dl_bytes(dl, url, temp_fd, zck->comp_index_size, start, + &buffer_len)) + return False; + if(!zck_read_index(zck, temp_fd)) + return False; + return True; +} + +size_t zck_dl_get_bytes_downloaded(zckDL *dl) { + VALIDATE(dl); + return dl->dl; +} + +size_t zck_dl_get_bytes_uploaded(zckDL *dl) { + VALIDATE(dl); + return dl->ul; +} + +int zck_dl_get_index(zckDL *dl, char *url) { + VALIDATE(dl); + return True; +} + +void zck_dl_global_init() { + curl_global_init(CURL_GLOBAL_ALL); +} + +void zck_dl_global_cleanup() { + curl_global_cleanup(); +} diff --git a/src/lib/dl/meson.build b/src/lib/dl/meson.build new file mode 100644 index 0000000..07a43e7 --- /dev/null +++ b/src/lib/dl/meson.build @@ -0,0 +1 @@ +sources += ['dl/range.c', 'dl/dl.c'] diff --git a/src/lib/dl/multipart_parser.c b/src/lib/dl/multipart_parser.c new file mode 100644 index 0000000..311acdc --- /dev/null +++ b/src/lib/dl/multipart_parser.c @@ -0,0 +1,306 @@ +/* Based on node-formidable by Felix Geisendörfer + * Igor Afonov - afonov@gmail.com - 2012 + * MIT License - http://www.opensource.org/licenses/mit-license.php + */ + +#include "multipart_parser.h" + +#include +#include +#include + +static void multipart_log(const char * format, ...) +{ +#ifdef DEBUG_MULTIPART + va_list args; + va_start(args, format); + + fprintf(stderr, "[HTTP_MULTIPART_PARSER] %s:%d: ", __FILE__, __LINE__); + vfprintf(stderr, format, args); + fprintf(stderr, "\n"); +#endif +} + +#define NOTIFY_CB(FOR) \ +do { \ + if (p->settings->on_##FOR) { \ + if (p->settings->on_##FOR(p) != 0) { \ + return i; \ + } \ + } \ +} while (0) + +#define EMIT_DATA_CB(FOR, ptr, len) \ +do { \ + if (p->settings->on_##FOR) { \ + if (p->settings->on_##FOR(p, ptr, len) != 0) { \ + return i; \ + } \ + } \ +} while (0) + + +#define LF 10 +#define CR 13 + +struct multipart_parser { + void * data; + + size_t index; + size_t boundary_length; + + unsigned char state; + + const multipart_parser_settings* settings; + + char* lookbehind; + char multipart_boundary[1]; +}; + +enum state { + s_uninitialized = 1, + s_start, + s_start_boundary, + s_header_field_start, + s_header_field, + s_headers_almost_done, + s_header_value_start, + s_header_value, + s_header_value_almost_done, + s_part_data_start, + s_part_data, + s_part_data_almost_boundary, + s_part_data_boundary, + s_part_data_almost_end, + s_part_data_end, + s_part_data_final_hyphen, + s_end +}; + +multipart_parser* multipart_parser_init + (const char *boundary, const multipart_parser_settings* settings) { + + multipart_parser* p = malloc(sizeof(multipart_parser) + + strlen(boundary) + + strlen(boundary) + 9); + + strcpy(p->multipart_boundary, boundary); + p->boundary_length = strlen(boundary); + + p->lookbehind = (p->multipart_boundary + p->boundary_length + 1); + + p->index = 0; + p->state = s_start; + p->settings = settings; + + return p; +} + +void multipart_parser_free(multipart_parser* p) { + free(p); +} + +void multipart_parser_set_data(multipart_parser *p, void *data) { + p->data = data; +} + +void *multipart_parser_get_data(multipart_parser *p) { + return p->data; +} + +size_t multipart_parser_execute(multipart_parser* p, const char *buf, size_t len) { + size_t i = 0; + size_t mark = 0; + char c, cl; + int is_last = 0; + + while(i < len) { + c = buf[i]; + is_last = (i == (len - 1)); + switch (p->state) { + case s_start: + multipart_log("s_start"); + p->index = 0; + p->state = s_start_boundary; + + /* fallthrough */ + case s_start_boundary: + multipart_log("s_start_boundary"); + if (p->index == p->boundary_length) { + if (c != CR) { + return i; + } + p->index++; + break; + } else if (p->index == (p->boundary_length + 1)) { + if (c != LF) { + return i; + } + p->index = 0; + NOTIFY_CB(part_data_begin); + p->state = s_header_field_start; + break; + } + if (c != p->multipart_boundary[p->index]) { + return i; + } + p->index++; + break; + + case s_header_field_start: + multipart_log("s_header_field_start"); + mark = i; + p->state = s_header_field; + + /* fallthrough */ + case s_header_field: + multipart_log("s_header_field"); + if (c == CR) { + p->state = s_headers_almost_done; + break; + } + + if (c == ':') { + EMIT_DATA_CB(header_field, buf + mark, i - mark); + p->state = s_header_value_start; + break; + } + + cl = tolower(c); + if ((c != '-') && (cl < 'a' || cl > 'z')) { + multipart_log("invalid character in header name"); + return i; + } + if (is_last) + EMIT_DATA_CB(header_field, buf + mark, (i - mark) + 1); + break; + + case s_headers_almost_done: + multipart_log("s_headers_almost_done"); + if (c != LF) { + return i; + } + + p->state = s_part_data_start; + break; + + case s_header_value_start: + multipart_log("s_header_value_start"); + if (c == ' ') { + break; + } + + mark = i; + p->state = s_header_value; + + /* fallthrough */ + case s_header_value: + multipart_log("s_header_value"); + if (c == CR) { + EMIT_DATA_CB(header_value, buf + mark, i - mark); + p->state = s_header_value_almost_done; + break; + } + if (is_last) + EMIT_DATA_CB(header_value, buf + mark, (i - mark) + 1); + break; + + case s_header_value_almost_done: + multipart_log("s_header_value_almost_done"); + if (c != LF) { + return i; + } + p->state = s_header_field_start; + break; + + case s_part_data_start: + multipart_log("s_part_data_start"); + NOTIFY_CB(headers_complete); + mark = i; + p->state = s_part_data; + + /* fallthrough */ + case s_part_data: + multipart_log("s_part_data"); + if (c == CR) { + EMIT_DATA_CB(part_data, buf + mark, i - mark); + mark = i; + p->state = s_part_data_almost_boundary; + p->lookbehind[0] = CR; + break; + } + if (is_last) + EMIT_DATA_CB(part_data, buf + mark, (i - mark) + 1); + break; + + case s_part_data_almost_boundary: + multipart_log("s_part_data_almost_boundary"); + if (c == LF) { + p->state = s_part_data_boundary; + p->lookbehind[1] = LF; + p->index = 0; + break; + } + EMIT_DATA_CB(part_data, p->lookbehind, 1); + p->state = s_part_data; + mark = i --; + break; + + case s_part_data_boundary: + multipart_log("s_part_data_boundary"); + if (p->multipart_boundary[p->index] != c) { + EMIT_DATA_CB(part_data, p->lookbehind, 2 + p->index); + p->state = s_part_data; + mark = i --; + break; + } + p->lookbehind[2 + p->index] = c; + if ((++ p->index) == p->boundary_length) { + NOTIFY_CB(part_data_end); + p->state = s_part_data_almost_end; + } + break; + + case s_part_data_almost_end: + multipart_log("s_part_data_almost_end"); + if (c == '-') { + p->state = s_part_data_final_hyphen; + break; + } + if (c == CR) { + p->state = s_part_data_end; + break; + } + return i; + + case s_part_data_final_hyphen: + multipart_log("s_part_data_final_hyphen"); + if (c == '-') { + NOTIFY_CB(body_end); + p->state = s_end; + break; + } + return i; + + case s_part_data_end: + multipart_log("s_part_data_end"); + if (c == LF) { + p->state = s_header_field_start; + NOTIFY_CB(part_data_begin); + break; + } + return i; + + case s_end: + multipart_log("s_end: %02X", (int) c); + break; + + default: + multipart_log("Multipart parser unrecoverable error"); + return 0; + } + ++ i; + } + + return len; +} diff --git a/src/lib/dl/multipart_parser.h b/src/lib/dl/multipart_parser.h new file mode 100644 index 0000000..87e67f4 --- /dev/null +++ b/src/lib/dl/multipart_parser.h @@ -0,0 +1,48 @@ +/* Based on node-formidable by Felix Geisendörfer + * Igor Afonov - afonov@gmail.com - 2012 + * MIT License - http://www.opensource.org/licenses/mit-license.php + */ +#ifndef _multipart_parser_h +#define _multipart_parser_h + +#ifdef __cplusplus +extern "C" +{ +#endif + +#include +#include + +typedef struct multipart_parser multipart_parser; +typedef struct multipart_parser_settings multipart_parser_settings; +typedef struct multipart_parser_state multipart_parser_state; + +typedef int (*multipart_data_cb) (multipart_parser*, const char *at, size_t length); +typedef int (*multipart_notify_cb) (multipart_parser*); + +struct multipart_parser_settings { + multipart_data_cb on_header_field; + multipart_data_cb on_header_value; + multipart_data_cb on_part_data; + + multipart_notify_cb on_part_data_begin; + multipart_notify_cb on_headers_complete; + multipart_notify_cb on_part_data_end; + multipart_notify_cb on_body_end; +}; + +multipart_parser* multipart_parser_init + (const char *boundary, const multipart_parser_settings* settings); + +void multipart_parser_free(multipart_parser* p); + +size_t multipart_parser_execute(multipart_parser* p, const char *buf, size_t len); + +void multipart_parser_set_data(multipart_parser* p, void* data); +void * multipart_parser_get_data(multipart_parser* p); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif diff --git a/src/lib/dl/range.c b/src/lib/dl/range.c new file mode 100644 index 0000000..2c29d6b --- /dev/null +++ b/src/lib/dl/range.c @@ -0,0 +1,229 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "zck_private.h" + +void zck_range_remove(zckRange *range) { + if(range->prev) + range->prev->next = range->next; + if(range->next) + range->next->prev = range->prev; + free(range); +} + +void zck_range_close(zckRangeInfo *info) { + zckRange *next = info->first; + while(next) { + zckRange *tmp = next; + next = next->next; + free(tmp); + } +} + +zckRange *zck_range_insert_new(zckRange *prev, zckRange *next, uint64_t start, uint64_t end) { + zckRange *new = zmalloc(sizeof(zckRange)); + if(!new) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckRange)); + return NULL; + } + new->start = start; + new->end = end; + if(prev) { + new->prev = prev; + prev->next = new; + } + if(next) { + new->next = next; + next->prev = new; + } + return new; +} + +void zck_range_merge_combined(zckRangeInfo *info) { + if(!info) { + zck_log(ZCK_LOG_ERROR, "zckRangeInfo not allocated\n"); + return; + } + for(zckRange *ptr=info->first; ptr;) { + if(ptr->next && ptr->end >= ptr->next->start-1) { + if(ptr->end < ptr->next->end) + ptr->end = ptr->next->end; + zck_range_remove(ptr->next); + info->count -= 1; + } else { + ptr = ptr->next; + } + } +} + +int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end) { + if(!info) { + zck_log(ZCK_LOG_ERROR, "zckRangeInfo not allocated\n"); + return False; + } + zckRange *prev = info->first; + for(zckRange *ptr=info->first; ptr;) { + prev = ptr; + if(start > ptr->start) { + ptr = ptr->next; + continue; + } else if(start < ptr->start) { + if(zck_range_insert_new(ptr->prev, ptr, start, end) == NULL) + return False; + if(info->first == ptr) { + info->first = ptr->prev; + } + info->count += 1; + zck_range_merge_combined(info); + return True; + } else { // start == ptr->start + if(end > ptr->end) + ptr->end = end; + info->count += 1; + zck_range_merge_combined(info); + return True; + } + } + /* We've only reached here if we should be last item */ + zckRange *new = zck_range_insert_new(prev, NULL, start, end); + if(new == NULL) + return False; + if(info->first == NULL) + info->first = new; + info->count += 1; + zck_range_merge_combined(info); + return True; +} + +int zck_range_calc_segments(zckRangeInfo *info, unsigned int max_ranges) { + if(max_ranges == 0) + return False; + info->segments = (info->count + max_ranges - 1) / max_ranges; + info->max_ranges = max_ranges; + return True; +} + +int zck_range_get_need_dl(zckRangeInfo *info, zckCtx *zck_src, zckCtx *zck_tgt) { + zckIndex *tgt_idx = zck_get_index(zck_tgt); + zckIndex *src_idx = zck_get_index(zck_src); + while(tgt_idx) { + int found = False; + src_idx = zck_get_index(zck_src); + + while(src_idx) { + if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) == 0) { + found = True; + break; + } + src_idx = src_idx->next; + } + if(!found) + if(!zck_range_add(info, tgt_idx->start, tgt_idx->start+tgt_idx->length-1)) + return False; + + tgt_idx = tgt_idx->next; + } + return True; +} + +char *zck_range_get_char(zckRange **range, int max_ranges) { + int buf_size=32768; + char *output=malloc(buf_size); + if(!output) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", buf_size); + return NULL; + } + + if(snprintf(output, 20, "Range: bytes=") > 20) { + free(output); + zck_log(ZCK_LOG_ERROR, "Error writing range\n"); + return NULL; + } + int loc = strlen(output); + int count=0; + while(*range) { + int length = snprintf(output+loc, buf_size-loc, "%lu-%lu,", + (*range)->start, (*range)->end); + if(length < 0) { + zck_log(ZCK_LOG_ERROR, "Unable to get range: %s\n", + strerror(errno)); + free(output); + return NULL; + } + if(length > buf_size-loc) { + buf_size = (int)(buf_size * 1.5); + output = realloc(output, buf_size); + if(output == NULL) { + free(output); + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + buf_size); + return NULL; + } + continue; + } + loc += length; + count++; + *range = (*range)->next; + if(count == max_ranges) + break; + } + output[loc-1]='\0'; // Remove final comma + output = realloc(output, loc); + if(output == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to shrink range to %lu bytes\n", + loc); + free(output); + return NULL; + } + return output; +} + +int zck_range_get_array(zckRangeInfo *info, char **ra) { + if(!info) { + zck_log(ZCK_LOG_ERROR, "zckRangeInfo not allocated\n"); + return False; + } + zckRange *tmp = info->first; + for(int i=0; isegments; i++) { + ra[i] = zck_range_get_char(&tmp, info->max_ranges); + if(ra[i] == NULL) + return False; + } + return True; +} + +int zck_range_get_max(zckRangeInfo *info, char *url) { + return True; +} diff --git a/src/lib/hash/hash.c b/src/lib/hash/hash.c new file mode 100644 index 0000000..3555242 --- /dev/null +++ b/src/lib/hash/hash.c @@ -0,0 +1,147 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "zck_private.h" +#include "sha1/sha1.h" +#include "sha2/sha2.h" + +static char unknown[] = "Unknown(\0\0\0\0\0"; + +const static char *HASH_NAME[] = { + "SHA-1", + "SHA-256" +}; + +int zck_hash_setup(zckHashType *ht, int h) { + if(ht) { + if(h == ZCK_HASH_SHA1) { + memset(ht, 0, sizeof(zckHashType)); + ht->type = ZCK_HASH_SHA1; + ht->digest_size = SHA1_DIGEST_LENGTH; + return True; + }else if(h == ZCK_HASH_SHA256) { + memset(ht, 0, sizeof(zckHashType)); + ht->type = ZCK_HASH_SHA256; + ht->digest_size = SHA256_DIGEST_SIZE; + return True; + } + zck_log(ZCK_LOG_ERROR, "Unsupported hash type: %s\n", + zck_hash_name_from_type(h)); + return False; + } + zck_log(ZCK_LOG_ERROR, "zckHashType is null\n"); + return False; +} + +int zck_hash_init(zckHash *hash, zckHashType *hash_type) { + if(hash && hash_type) { + if(hash_type->type == ZCK_HASH_SHA1) { + zck_log(ZCK_LOG_DEBUG, "Initializing SHA-1 hash\n"); + hash->ctx = zmalloc(sizeof(SHA_CTX)); + hash->type = hash_type; + if(hash->ctx == NULL) + return False; + SHA1_Init((SHA_CTX *) hash->ctx); + return True; + }else if(hash_type->type == ZCK_HASH_SHA256) { + zck_log(ZCK_LOG_DEBUG, "Initializing SHA-256 hash\n"); + hash->ctx = zmalloc(sizeof(sha256_ctx)); + hash->type = hash_type; + if(hash->ctx == NULL) + return False; + sha256_init((sha256_ctx *) hash->ctx); + return True; + } + zck_log(ZCK_LOG_ERROR, "Unsupported hash type: %i\n", hash_type->type); + return False; + } + zck_log(ZCK_LOG_ERROR, "Either zckHash or zckHashType struct is null\n"); + return False; +} + +int zck_hash_update(zckHash *hash, const char *message, const size_t size) { + if(hash && hash->ctx && hash->type) { + if(hash->type->type == ZCK_HASH_SHA1) { + SHA1_Update((SHA_CTX *)hash->ctx, (const sha1_byte *)message, size); + return True; + } else if(hash->type->type == ZCK_HASH_SHA256) { + sha256_update((sha256_ctx *)hash->ctx, (const unsigned char *)message, size); + return True; + } + zck_log(ZCK_LOG_ERROR, "Unsupported hash type: %i\n", hash->type); + return False; + } + zck_log(ZCK_LOG_ERROR, "Hash hasn't been initialized\n"); + return False; +} + +void zck_hash_close(zckHash *hash) { + if(!hash) + return; + + if(hash->ctx) { + free(hash->ctx); + hash->ctx = NULL; + } + hash->type = NULL; + return; +} + +char *zck_hash_finalize(zckHash *hash) { + if(hash && hash->ctx && hash->type) { + if(hash->type->type == ZCK_HASH_SHA1) { + unsigned char *digest = zmalloc(hash->type->digest_size); + SHA1_Final((sha1_byte*)digest, (SHA_CTX *)hash->ctx); + zck_hash_close(hash); + return (char *)digest; + } else if(hash->type->type == ZCK_HASH_SHA256) { + unsigned char *digest = zmalloc(hash->type->digest_size); + sha256_final((sha256_ctx *)hash->ctx, digest); + zck_hash_close(hash); + return (char *)digest; + } + zck_log(ZCK_LOG_ERROR, "Unsupported hash type: %i\n", hash->type); + return NULL; + } + zck_log(ZCK_LOG_ERROR, "Hash hasn't been initialized\n"); + return NULL; +} + +const char *zck_hash_name_from_type(uint8_t hash_type) { + if(hash_type > 1) { + snprintf(unknown+8, 4, "%i)", hash_type); + return unknown; + } + return HASH_NAME[hash_type]; +} + + diff --git a/src/lib/hash/meson.build b/src/lib/hash/meson.build new file mode 100644 index 0000000..a70a9d7 --- /dev/null +++ b/src/lib/hash/meson.build @@ -0,0 +1,3 @@ +sources += ['hash/hash.c'] +subdir('sha1') +subdir('sha2') diff --git a/src/lib/hash/sha1/meson.build b/src/lib/hash/sha1/meson.build new file mode 100644 index 0000000..b75c4f7 --- /dev/null +++ b/src/lib/hash/sha1/meson.build @@ -0,0 +1 @@ +sources += ['hash/sha1/sha1.c'] diff --git a/src/lib/hash/sha1/sha1.c b/src/lib/hash/sha1/sha1.c new file mode 100644 index 0000000..e34de61 --- /dev/null +++ b/src/lib/hash/sha1/sha1.c @@ -0,0 +1,157 @@ +/* + * sha1.c + * + * Originally witten by Steve Reid + * + * Modified by Aaron D. Gifford + * + * NO COPYRIGHT - THIS IS 100% IN THE PUBLIC DOMAIN + * + * The original unmodified version is available at: + * ftp://ftp.funet.fi/pub/crypt/hash/sha/sha1.c + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR(S) OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include "sha1.h" + +#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) + +/* blk0() and blk() perform the initial expand. */ +/* I got the idea of expanding during the round function from SSLeay */ + +#ifdef LITTLE_ENDIAN +#define blk0(i) (block->l[i] = (rol(block->l[i],24)&(sha1_quadbyte)0xFF00FF00) \ + |(rol(block->l[i],8)&(sha1_quadbyte)0x00FF00FF)) +#else +#define blk0(i) block->l[i] +#endif + +#define blk(i) (block->l[i&15] = rol(block->l[(i+13)&15]^block->l[(i+8)&15] \ + ^block->l[(i+2)&15]^block->l[i&15],1)) + +/* (R0+R1), R2, R3, R4 are the different operations used in SHA1 */ +#define R0(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk0(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R1(v,w,x,y,z,i) z+=((w&(x^y))^y)+blk(i)+0x5A827999+rol(v,5);w=rol(w,30); +#define R2(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0x6ED9EBA1+rol(v,5);w=rol(w,30); +#define R3(v,w,x,y,z,i) z+=(((w|x)&y)|(w&x))+blk(i)+0x8F1BBCDC+rol(v,5);w=rol(w,30); +#define R4(v,w,x,y,z,i) z+=(w^x^y)+blk(i)+0xCA62C1D6+rol(v,5);w=rol(w,30); + +typedef union _BYTE64QUAD16 { + sha1_byte c[64]; + sha1_quadbyte l[16]; +} BYTE64QUAD16; + +/* Hash a single 512-bit block. This is the core of the algorithm. */ +void SHA1_Transform(sha1_quadbyte state[5], const sha1_byte buffer[64]) { + sha1_quadbyte a, b, c, d, e; + BYTE64QUAD16 block[64]; + + memcpy(block, buffer, 64); + /* Copy context->state[] to working vars */ + a = state[0]; + b = state[1]; + c = state[2]; + d = state[3]; + e = state[4]; + /* 4 rounds of 20 operations each. Loop unrolled. */ + R0(a,b,c,d,e, 0); R0(e,a,b,c,d, 1); R0(d,e,a,b,c, 2); R0(c,d,e,a,b, 3); + R0(b,c,d,e,a, 4); R0(a,b,c,d,e, 5); R0(e,a,b,c,d, 6); R0(d,e,a,b,c, 7); + R0(c,d,e,a,b, 8); R0(b,c,d,e,a, 9); R0(a,b,c,d,e,10); R0(e,a,b,c,d,11); + R0(d,e,a,b,c,12); R0(c,d,e,a,b,13); R0(b,c,d,e,a,14); R0(a,b,c,d,e,15); + R1(e,a,b,c,d,16); R1(d,e,a,b,c,17); R1(c,d,e,a,b,18); R1(b,c,d,e,a,19); + R2(a,b,c,d,e,20); R2(e,a,b,c,d,21); R2(d,e,a,b,c,22); R2(c,d,e,a,b,23); + R2(b,c,d,e,a,24); R2(a,b,c,d,e,25); R2(e,a,b,c,d,26); R2(d,e,a,b,c,27); + R2(c,d,e,a,b,28); R2(b,c,d,e,a,29); R2(a,b,c,d,e,30); R2(e,a,b,c,d,31); + R2(d,e,a,b,c,32); R2(c,d,e,a,b,33); R2(b,c,d,e,a,34); R2(a,b,c,d,e,35); + R2(e,a,b,c,d,36); R2(d,e,a,b,c,37); R2(c,d,e,a,b,38); R2(b,c,d,e,a,39); + R3(a,b,c,d,e,40); R3(e,a,b,c,d,41); R3(d,e,a,b,c,42); R3(c,d,e,a,b,43); + R3(b,c,d,e,a,44); R3(a,b,c,d,e,45); R3(e,a,b,c,d,46); R3(d,e,a,b,c,47); + R3(c,d,e,a,b,48); R3(b,c,d,e,a,49); R3(a,b,c,d,e,50); R3(e,a,b,c,d,51); + R3(d,e,a,b,c,52); R3(c,d,e,a,b,53); R3(b,c,d,e,a,54); R3(a,b,c,d,e,55); + R3(e,a,b,c,d,56); R3(d,e,a,b,c,57); R3(c,d,e,a,b,58); R3(b,c,d,e,a,59); + R4(a,b,c,d,e,60); R4(e,a,b,c,d,61); R4(d,e,a,b,c,62); R4(c,d,e,a,b,63); + R4(b,c,d,e,a,64); R4(a,b,c,d,e,65); R4(e,a,b,c,d,66); R4(d,e,a,b,c,67); + R4(c,d,e,a,b,68); R4(b,c,d,e,a,69); R4(a,b,c,d,e,70); R4(e,a,b,c,d,71); + R4(d,e,a,b,c,72); R4(c,d,e,a,b,73); R4(b,c,d,e,a,74); R4(a,b,c,d,e,75); + R4(e,a,b,c,d,76); R4(d,e,a,b,c,77); R4(c,d,e,a,b,78); R4(b,c,d,e,a,79); + /* Add the working vars back into context.state[] */ + state[0] += a; + state[1] += b; + state[2] += c; + state[3] += d; + state[4] += e; + /* Wipe variables */ + a = b = c = d = e = 0; +} + + +/* SHA1_Init - Initialize new context */ +void SHA1_Init(SHA_CTX* context) { + /* SHA1 initialization constants */ + context->state[0] = 0x67452301; + context->state[1] = 0xEFCDAB89; + context->state[2] = 0x98BADCFE; + context->state[3] = 0x10325476; + context->state[4] = 0xC3D2E1F0; + context->count[0] = context->count[1] = 0; +} + +/* Run your data through this. */ +void SHA1_Update(SHA_CTX *context, const sha1_byte *data, unsigned int len) { + unsigned int i, j; + j = (context->count[0] >> 3) & 63; + if ((context->count[0] += len << 3) < (len << 3)) context->count[1]++; + context->count[1] += (len >> 29); + if ((j + len) > 63) { + memcpy(&context->buffer[j], data, (i = 64-j)); + SHA1_Transform(context->state, context->buffer); + for ( ; i + 63 < len; i += 64) { + SHA1_Transform(context->state, &data[i]); + } + j = 0; + } + else i = 0; + memcpy(&context->buffer[j], &data[i], len - i); +} + + +/* Add padding and return the message digest. */ +void SHA1_Final(sha1_byte digest[SHA1_DIGEST_LENGTH], SHA_CTX *context) { + sha1_quadbyte i, j; + sha1_byte finalcount[8]; + + for (i = 0; i < 8; i++) { + finalcount[i] = (sha1_byte)((context->count[(i >= 4 ? 0 : 1)] + >> ((3-(i & 3)) * 8) ) & 255); /* Endian independent */ + } + SHA1_Update(context, (sha1_byte *)"\200", 1); + while ((context->count[0] & 504) != 448) { + SHA1_Update(context, (sha1_byte *)"\0", 1); + } + /* Should cause a SHA1_Transform() */ + SHA1_Update(context, finalcount, 8); + for (i = 0; i < SHA1_DIGEST_LENGTH; i++) { + digest[i] = (sha1_byte) + ((context->state[i>>2] >> ((3-(i & 3)) * 8) ) & 255); + } + /* Wipe variables */ + i = j = 0; + memset(context->buffer, 0, SHA1_BLOCK_LENGTH); + memset(context->state, 0, SHA1_DIGEST_LENGTH); + memset(context->count, 0, 8); + memset(&finalcount, 0, 8); +} + diff --git a/src/lib/hash/sha1/sha1.h b/src/lib/hash/sha1/sha1.h new file mode 100644 index 0000000..300a466 --- /dev/null +++ b/src/lib/hash/sha1/sha1.h @@ -0,0 +1,62 @@ +/* + * sha.h + * + * Originally taken from the public domain SHA1 implementation + * written by by Steve Reid + * + * Modified by Aaron D. Gifford + * + * NO COPYRIGHT - THIS IS 100% IN THE PUBLIC DOMAIN + * + * The original unmodified version is available at: + * ftp://ftp.funet.fi/pub/crypt/hash/sha/sha1.c + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR(S) AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR(S) OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef __SHA1_H__ +#define __SHA1_H__ + +#include + +#if __BYTE_ORDER == __LITTLE_ENDIAN && !defined(LITTLE_ENDIAN) +#define LITTLE_ENDIAN +#endif + +/* Make sure you define these types for your architecture: */ +typedef uint32_t sha1_quadbyte; /* 4 byte type */ +typedef char sha1_byte; /* single byte type */ + +/* + * Be sure to get the above definitions right. For instance, on my + * x86 based FreeBSD box, I define LITTLE_ENDIAN and use the type + * "unsigned long" for the quadbyte. On FreeBSD on the Alpha, however, + * while I still use LITTLE_ENDIAN, I must define the quadbyte type + * as "unsigned int" instead. + */ + +#define SHA1_BLOCK_LENGTH 64 +#define SHA1_DIGEST_LENGTH 20 + +/* The SHA1 structure: */ +typedef struct _SHA_CTX { + sha1_quadbyte state[5]; + sha1_quadbyte count[2]; + sha1_byte buffer[SHA1_BLOCK_LENGTH]; +} SHA_CTX; + +void SHA1_Init(SHA_CTX *context); +void SHA1_Update(SHA_CTX *context, const sha1_byte *data, unsigned int len); +void SHA1_Final(sha1_byte digest[SHA1_DIGEST_LENGTH], SHA_CTX* context); + +#endif diff --git a/src/lib/hash/sha2/meson.build b/src/lib/hash/sha2/meson.build new file mode 100644 index 0000000..c2c4b29 --- /dev/null +++ b/src/lib/hash/sha2/meson.build @@ -0,0 +1 @@ +sources += ['hash/sha2/sha2.c'] diff --git a/src/lib/hash/sha2/sha2.c b/src/lib/hash/sha2/sha2.c new file mode 100644 index 0000000..0fe4c37 --- /dev/null +++ b/src/lib/hash/sha2/sha2.c @@ -0,0 +1,949 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#if 0 +#define UNROLL_LOOPS /* Enable loops unrolling */ +#endif + +#include + +#include "sha2.h" + +#define SHFR(x, n) (x >> n) +#define ROTR(x, n) ((x >> n) | (x << ((sizeof(x) << 3) - n))) +#define ROTL(x, n) ((x << n) | (x >> ((sizeof(x) << 3) - n))) +#define CH(x, y, z) ((x & y) ^ (~x & z)) +#define MAJ(x, y, z) ((x & y) ^ (x & z) ^ (y & z)) + +#define SHA256_F1(x) (ROTR(x, 2) ^ ROTR(x, 13) ^ ROTR(x, 22)) +#define SHA256_F2(x) (ROTR(x, 6) ^ ROTR(x, 11) ^ ROTR(x, 25)) +#define SHA256_F3(x) (ROTR(x, 7) ^ ROTR(x, 18) ^ SHFR(x, 3)) +#define SHA256_F4(x) (ROTR(x, 17) ^ ROTR(x, 19) ^ SHFR(x, 10)) + +#define SHA512_F1(x) (ROTR(x, 28) ^ ROTR(x, 34) ^ ROTR(x, 39)) +#define SHA512_F2(x) (ROTR(x, 14) ^ ROTR(x, 18) ^ ROTR(x, 41)) +#define SHA512_F3(x) (ROTR(x, 1) ^ ROTR(x, 8) ^ SHFR(x, 7)) +#define SHA512_F4(x) (ROTR(x, 19) ^ ROTR(x, 61) ^ SHFR(x, 6)) + +#define UNPACK32(x, str) \ +{ \ + *((str) + 3) = (uint8) ((x) ); \ + *((str) + 2) = (uint8) ((x) >> 8); \ + *((str) + 1) = (uint8) ((x) >> 16); \ + *((str) + 0) = (uint8) ((x) >> 24); \ +} + +#define PACK32(str, x) \ +{ \ + *(x) = ((uint32) *((str) + 3) ) \ + | ((uint32) *((str) + 2) << 8) \ + | ((uint32) *((str) + 1) << 16) \ + | ((uint32) *((str) + 0) << 24); \ +} + +#define UNPACK64(x, str) \ +{ \ + *((str) + 7) = (uint8) ((x) ); \ + *((str) + 6) = (uint8) ((x) >> 8); \ + *((str) + 5) = (uint8) ((x) >> 16); \ + *((str) + 4) = (uint8) ((x) >> 24); \ + *((str) + 3) = (uint8) ((x) >> 32); \ + *((str) + 2) = (uint8) ((x) >> 40); \ + *((str) + 1) = (uint8) ((x) >> 48); \ + *((str) + 0) = (uint8) ((x) >> 56); \ +} + +#define PACK64(str, x) \ +{ \ + *(x) = ((uint64) *((str) + 7) ) \ + | ((uint64) *((str) + 6) << 8) \ + | ((uint64) *((str) + 5) << 16) \ + | ((uint64) *((str) + 4) << 24) \ + | ((uint64) *((str) + 3) << 32) \ + | ((uint64) *((str) + 2) << 40) \ + | ((uint64) *((str) + 1) << 48) \ + | ((uint64) *((str) + 0) << 56); \ +} + +/* Macros used for loops unrolling */ + +#define SHA256_SCR(i) \ +{ \ + w[i] = SHA256_F4(w[i - 2]) + w[i - 7] \ + + SHA256_F3(w[i - 15]) + w[i - 16]; \ +} + +#define SHA512_SCR(i) \ +{ \ + w[i] = SHA512_F4(w[i - 2]) + w[i - 7] \ + + SHA512_F3(w[i - 15]) + w[i - 16]; \ +} + +#define SHA256_EXP(a, b, c, d, e, f, g, h, j) \ +{ \ + t1 = wv[h] + SHA256_F2(wv[e]) + CH(wv[e], wv[f], wv[g]) \ + + sha256_k[j] + w[j]; \ + t2 = SHA256_F1(wv[a]) + MAJ(wv[a], wv[b], wv[c]); \ + wv[d] += t1; \ + wv[h] = t1 + t2; \ +} + +#define SHA512_EXP(a, b, c, d, e, f, g ,h, j) \ +{ \ + t1 = wv[h] + SHA512_F2(wv[e]) + CH(wv[e], wv[f], wv[g]) \ + + sha512_k[j] + w[j]; \ + t2 = SHA512_F1(wv[a]) + MAJ(wv[a], wv[b], wv[c]); \ + wv[d] += t1; \ + wv[h] = t1 + t2; \ +} + +uint32 sha224_h0[8] = + {0xc1059ed8, 0x367cd507, 0x3070dd17, 0xf70e5939, + 0xffc00b31, 0x68581511, 0x64f98fa7, 0xbefa4fa4}; + +uint32 sha256_h0[8] = + {0x6a09e667, 0xbb67ae85, 0x3c6ef372, 0xa54ff53a, + 0x510e527f, 0x9b05688c, 0x1f83d9ab, 0x5be0cd19}; + +uint64 sha384_h0[8] = + {0xcbbb9d5dc1059ed8ULL, 0x629a292a367cd507ULL, + 0x9159015a3070dd17ULL, 0x152fecd8f70e5939ULL, + 0x67332667ffc00b31ULL, 0x8eb44a8768581511ULL, + 0xdb0c2e0d64f98fa7ULL, 0x47b5481dbefa4fa4ULL}; + +uint64 sha512_h0[8] = + {0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, + 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, + 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, + 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL}; + +uint32 sha256_k[64] = + {0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, + 0x3956c25b, 0x59f111f1, 0x923f82a4, 0xab1c5ed5, + 0xd807aa98, 0x12835b01, 0x243185be, 0x550c7dc3, + 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, 0xc19bf174, + 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, + 0x983e5152, 0xa831c66d, 0xb00327c8, 0xbf597fc7, + 0xc6e00bf3, 0xd5a79147, 0x06ca6351, 0x14292967, + 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, 0x53380d13, + 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, + 0xd192e819, 0xd6990624, 0xf40e3585, 0x106aa070, + 0x19a4c116, 0x1e376c08, 0x2748774c, 0x34b0bcb5, + 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, 0x682e6ff3, + 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2}; + +uint64 sha512_k[80] = + {0x428a2f98d728ae22ULL, 0x7137449123ef65cdULL, + 0xb5c0fbcfec4d3b2fULL, 0xe9b5dba58189dbbcULL, + 0x3956c25bf348b538ULL, 0x59f111f1b605d019ULL, + 0x923f82a4af194f9bULL, 0xab1c5ed5da6d8118ULL, + 0xd807aa98a3030242ULL, 0x12835b0145706fbeULL, + 0x243185be4ee4b28cULL, 0x550c7dc3d5ffb4e2ULL, + 0x72be5d74f27b896fULL, 0x80deb1fe3b1696b1ULL, + 0x9bdc06a725c71235ULL, 0xc19bf174cf692694ULL, + 0xe49b69c19ef14ad2ULL, 0xefbe4786384f25e3ULL, + 0x0fc19dc68b8cd5b5ULL, 0x240ca1cc77ac9c65ULL, + 0x2de92c6f592b0275ULL, 0x4a7484aa6ea6e483ULL, + 0x5cb0a9dcbd41fbd4ULL, 0x76f988da831153b5ULL, + 0x983e5152ee66dfabULL, 0xa831c66d2db43210ULL, + 0xb00327c898fb213fULL, 0xbf597fc7beef0ee4ULL, + 0xc6e00bf33da88fc2ULL, 0xd5a79147930aa725ULL, + 0x06ca6351e003826fULL, 0x142929670a0e6e70ULL, + 0x27b70a8546d22ffcULL, 0x2e1b21385c26c926ULL, + 0x4d2c6dfc5ac42aedULL, 0x53380d139d95b3dfULL, + 0x650a73548baf63deULL, 0x766a0abb3c77b2a8ULL, + 0x81c2c92e47edaee6ULL, 0x92722c851482353bULL, + 0xa2bfe8a14cf10364ULL, 0xa81a664bbc423001ULL, + 0xc24b8b70d0f89791ULL, 0xc76c51a30654be30ULL, + 0xd192e819d6ef5218ULL, 0xd69906245565a910ULL, + 0xf40e35855771202aULL, 0x106aa07032bbd1b8ULL, + 0x19a4c116b8d2d0c8ULL, 0x1e376c085141ab53ULL, + 0x2748774cdf8eeb99ULL, 0x34b0bcb5e19b48a8ULL, + 0x391c0cb3c5c95a63ULL, 0x4ed8aa4ae3418acbULL, + 0x5b9cca4f7763e373ULL, 0x682e6ff3d6b2b8a3ULL, + 0x748f82ee5defb2fcULL, 0x78a5636f43172f60ULL, + 0x84c87814a1f0ab72ULL, 0x8cc702081a6439ecULL, + 0x90befffa23631e28ULL, 0xa4506cebde82bde9ULL, + 0xbef9a3f7b2c67915ULL, 0xc67178f2e372532bULL, + 0xca273eceea26619cULL, 0xd186b8c721c0c207ULL, + 0xeada7dd6cde0eb1eULL, 0xf57d4f7fee6ed178ULL, + 0x06f067aa72176fbaULL, 0x0a637dc5a2c898a6ULL, + 0x113f9804bef90daeULL, 0x1b710b35131c471bULL, + 0x28db77f523047d84ULL, 0x32caab7b40c72493ULL, + 0x3c9ebe0a15c9bebcULL, 0x431d67c49c100d4cULL, + 0x4cc5d4becb3e42b6ULL, 0x597f299cfc657e2aULL, + 0x5fcb6fab3ad6faecULL, 0x6c44198c4a475817ULL}; + +/* SHA-256 functions */ + +void sha256_transf(sha256_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint32 w[64]; + uint32 wv[8]; + uint32 t1, t2; + const unsigned char *sub_block; + int i; + +#ifndef UNROLL_LOOPS + int j; +#endif + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 6); + +#ifndef UNROLL_LOOPS + for (j = 0; j < 16; j++) { + PACK32(&sub_block[j << 2], &w[j]); + } + + for (j = 16; j < 64; j++) { + SHA256_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 64; j++) { + t1 = wv[7] + SHA256_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha256_k[j] + w[j]; + t2 = SHA256_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } +#else + PACK32(&sub_block[ 0], &w[ 0]); PACK32(&sub_block[ 4], &w[ 1]); + PACK32(&sub_block[ 8], &w[ 2]); PACK32(&sub_block[12], &w[ 3]); + PACK32(&sub_block[16], &w[ 4]); PACK32(&sub_block[20], &w[ 5]); + PACK32(&sub_block[24], &w[ 6]); PACK32(&sub_block[28], &w[ 7]); + PACK32(&sub_block[32], &w[ 8]); PACK32(&sub_block[36], &w[ 9]); + PACK32(&sub_block[40], &w[10]); PACK32(&sub_block[44], &w[11]); + PACK32(&sub_block[48], &w[12]); PACK32(&sub_block[52], &w[13]); + PACK32(&sub_block[56], &w[14]); PACK32(&sub_block[60], &w[15]); + + SHA256_SCR(16); SHA256_SCR(17); SHA256_SCR(18); SHA256_SCR(19); + SHA256_SCR(20); SHA256_SCR(21); SHA256_SCR(22); SHA256_SCR(23); + SHA256_SCR(24); SHA256_SCR(25); SHA256_SCR(26); SHA256_SCR(27); + SHA256_SCR(28); SHA256_SCR(29); SHA256_SCR(30); SHA256_SCR(31); + SHA256_SCR(32); SHA256_SCR(33); SHA256_SCR(34); SHA256_SCR(35); + SHA256_SCR(36); SHA256_SCR(37); SHA256_SCR(38); SHA256_SCR(39); + SHA256_SCR(40); SHA256_SCR(41); SHA256_SCR(42); SHA256_SCR(43); + SHA256_SCR(44); SHA256_SCR(45); SHA256_SCR(46); SHA256_SCR(47); + SHA256_SCR(48); SHA256_SCR(49); SHA256_SCR(50); SHA256_SCR(51); + SHA256_SCR(52); SHA256_SCR(53); SHA256_SCR(54); SHA256_SCR(55); + SHA256_SCR(56); SHA256_SCR(57); SHA256_SCR(58); SHA256_SCR(59); + SHA256_SCR(60); SHA256_SCR(61); SHA256_SCR(62); SHA256_SCR(63); + + wv[0] = ctx->h[0]; wv[1] = ctx->h[1]; + wv[2] = ctx->h[2]; wv[3] = ctx->h[3]; + wv[4] = ctx->h[4]; wv[5] = ctx->h[5]; + wv[6] = ctx->h[6]; wv[7] = ctx->h[7]; + + SHA256_EXP(0,1,2,3,4,5,6,7, 0); SHA256_EXP(7,0,1,2,3,4,5,6, 1); + SHA256_EXP(6,7,0,1,2,3,4,5, 2); SHA256_EXP(5,6,7,0,1,2,3,4, 3); + SHA256_EXP(4,5,6,7,0,1,2,3, 4); SHA256_EXP(3,4,5,6,7,0,1,2, 5); + SHA256_EXP(2,3,4,5,6,7,0,1, 6); SHA256_EXP(1,2,3,4,5,6,7,0, 7); + SHA256_EXP(0,1,2,3,4,5,6,7, 8); SHA256_EXP(7,0,1,2,3,4,5,6, 9); + SHA256_EXP(6,7,0,1,2,3,4,5,10); SHA256_EXP(5,6,7,0,1,2,3,4,11); + SHA256_EXP(4,5,6,7,0,1,2,3,12); SHA256_EXP(3,4,5,6,7,0,1,2,13); + SHA256_EXP(2,3,4,5,6,7,0,1,14); SHA256_EXP(1,2,3,4,5,6,7,0,15); + SHA256_EXP(0,1,2,3,4,5,6,7,16); SHA256_EXP(7,0,1,2,3,4,5,6,17); + SHA256_EXP(6,7,0,1,2,3,4,5,18); SHA256_EXP(5,6,7,0,1,2,3,4,19); + SHA256_EXP(4,5,6,7,0,1,2,3,20); SHA256_EXP(3,4,5,6,7,0,1,2,21); + SHA256_EXP(2,3,4,5,6,7,0,1,22); SHA256_EXP(1,2,3,4,5,6,7,0,23); + SHA256_EXP(0,1,2,3,4,5,6,7,24); SHA256_EXP(7,0,1,2,3,4,5,6,25); + SHA256_EXP(6,7,0,1,2,3,4,5,26); SHA256_EXP(5,6,7,0,1,2,3,4,27); + SHA256_EXP(4,5,6,7,0,1,2,3,28); SHA256_EXP(3,4,5,6,7,0,1,2,29); + SHA256_EXP(2,3,4,5,6,7,0,1,30); SHA256_EXP(1,2,3,4,5,6,7,0,31); + SHA256_EXP(0,1,2,3,4,5,6,7,32); SHA256_EXP(7,0,1,2,3,4,5,6,33); + SHA256_EXP(6,7,0,1,2,3,4,5,34); SHA256_EXP(5,6,7,0,1,2,3,4,35); + SHA256_EXP(4,5,6,7,0,1,2,3,36); SHA256_EXP(3,4,5,6,7,0,1,2,37); + SHA256_EXP(2,3,4,5,6,7,0,1,38); SHA256_EXP(1,2,3,4,5,6,7,0,39); + SHA256_EXP(0,1,2,3,4,5,6,7,40); SHA256_EXP(7,0,1,2,3,4,5,6,41); + SHA256_EXP(6,7,0,1,2,3,4,5,42); SHA256_EXP(5,6,7,0,1,2,3,4,43); + SHA256_EXP(4,5,6,7,0,1,2,3,44); SHA256_EXP(3,4,5,6,7,0,1,2,45); + SHA256_EXP(2,3,4,5,6,7,0,1,46); SHA256_EXP(1,2,3,4,5,6,7,0,47); + SHA256_EXP(0,1,2,3,4,5,6,7,48); SHA256_EXP(7,0,1,2,3,4,5,6,49); + SHA256_EXP(6,7,0,1,2,3,4,5,50); SHA256_EXP(5,6,7,0,1,2,3,4,51); + SHA256_EXP(4,5,6,7,0,1,2,3,52); SHA256_EXP(3,4,5,6,7,0,1,2,53); + SHA256_EXP(2,3,4,5,6,7,0,1,54); SHA256_EXP(1,2,3,4,5,6,7,0,55); + SHA256_EXP(0,1,2,3,4,5,6,7,56); SHA256_EXP(7,0,1,2,3,4,5,6,57); + SHA256_EXP(6,7,0,1,2,3,4,5,58); SHA256_EXP(5,6,7,0,1,2,3,4,59); + SHA256_EXP(4,5,6,7,0,1,2,3,60); SHA256_EXP(3,4,5,6,7,0,1,2,61); + SHA256_EXP(2,3,4,5,6,7,0,1,62); SHA256_EXP(1,2,3,4,5,6,7,0,63); + + ctx->h[0] += wv[0]; ctx->h[1] += wv[1]; + ctx->h[2] += wv[2]; ctx->h[3] += wv[3]; + ctx->h[4] += wv[4]; ctx->h[5] += wv[5]; + ctx->h[6] += wv[6]; ctx->h[7] += wv[7]; +#endif /* !UNROLL_LOOPS */ + } +} + +void sha256(const unsigned char *message, unsigned int len, unsigned char *digest) +{ + sha256_ctx ctx; + + sha256_init(&ctx); + sha256_update(&ctx, message, len); + sha256_final(&ctx, digest); +} + +void sha256_init(sha256_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha256_h0[i]; + } +#else + ctx->h[0] = sha256_h0[0]; ctx->h[1] = sha256_h0[1]; + ctx->h[2] = sha256_h0[2]; ctx->h[3] = sha256_h0[3]; + ctx->h[4] = sha256_h0[4]; ctx->h[5] = sha256_h0[5]; + ctx->h[6] = sha256_h0[6]; ctx->h[7] = sha256_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA256_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA256_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA256_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA256_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha256_final(sha256_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA256_BLOCK_SIZE - 9) + < (ctx->len % SHA256_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 8; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +#else + UNPACK32(ctx->h[0], &digest[ 0]); + UNPACK32(ctx->h[1], &digest[ 4]); + UNPACK32(ctx->h[2], &digest[ 8]); + UNPACK32(ctx->h[3], &digest[12]); + UNPACK32(ctx->h[4], &digest[16]); + UNPACK32(ctx->h[5], &digest[20]); + UNPACK32(ctx->h[6], &digest[24]); + UNPACK32(ctx->h[7], &digest[28]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-512 functions */ + +void sha512_transf(sha512_ctx *ctx, const unsigned char *message, + unsigned int block_nb) +{ + uint64 w[80]; + uint64 wv[8]; + uint64 t1, t2; + const unsigned char *sub_block; + int i, j; + + for (i = 0; i < (int) block_nb; i++) { + sub_block = message + (i << 7); + +#ifndef UNROLL_LOOPS + for (j = 0; j < 16; j++) { + PACK64(&sub_block[j << 3], &w[j]); + } + + for (j = 16; j < 80; j++) { + SHA512_SCR(j); + } + + for (j = 0; j < 8; j++) { + wv[j] = ctx->h[j]; + } + + for (j = 0; j < 80; j++) { + t1 = wv[7] + SHA512_F2(wv[4]) + CH(wv[4], wv[5], wv[6]) + + sha512_k[j] + w[j]; + t2 = SHA512_F1(wv[0]) + MAJ(wv[0], wv[1], wv[2]); + wv[7] = wv[6]; + wv[6] = wv[5]; + wv[5] = wv[4]; + wv[4] = wv[3] + t1; + wv[3] = wv[2]; + wv[2] = wv[1]; + wv[1] = wv[0]; + wv[0] = t1 + t2; + } + + for (j = 0; j < 8; j++) { + ctx->h[j] += wv[j]; + } +#else + PACK64(&sub_block[ 0], &w[ 0]); PACK64(&sub_block[ 8], &w[ 1]); + PACK64(&sub_block[ 16], &w[ 2]); PACK64(&sub_block[ 24], &w[ 3]); + PACK64(&sub_block[ 32], &w[ 4]); PACK64(&sub_block[ 40], &w[ 5]); + PACK64(&sub_block[ 48], &w[ 6]); PACK64(&sub_block[ 56], &w[ 7]); + PACK64(&sub_block[ 64], &w[ 8]); PACK64(&sub_block[ 72], &w[ 9]); + PACK64(&sub_block[ 80], &w[10]); PACK64(&sub_block[ 88], &w[11]); + PACK64(&sub_block[ 96], &w[12]); PACK64(&sub_block[104], &w[13]); + PACK64(&sub_block[112], &w[14]); PACK64(&sub_block[120], &w[15]); + + SHA512_SCR(16); SHA512_SCR(17); SHA512_SCR(18); SHA512_SCR(19); + SHA512_SCR(20); SHA512_SCR(21); SHA512_SCR(22); SHA512_SCR(23); + SHA512_SCR(24); SHA512_SCR(25); SHA512_SCR(26); SHA512_SCR(27); + SHA512_SCR(28); SHA512_SCR(29); SHA512_SCR(30); SHA512_SCR(31); + SHA512_SCR(32); SHA512_SCR(33); SHA512_SCR(34); SHA512_SCR(35); + SHA512_SCR(36); SHA512_SCR(37); SHA512_SCR(38); SHA512_SCR(39); + SHA512_SCR(40); SHA512_SCR(41); SHA512_SCR(42); SHA512_SCR(43); + SHA512_SCR(44); SHA512_SCR(45); SHA512_SCR(46); SHA512_SCR(47); + SHA512_SCR(48); SHA512_SCR(49); SHA512_SCR(50); SHA512_SCR(51); + SHA512_SCR(52); SHA512_SCR(53); SHA512_SCR(54); SHA512_SCR(55); + SHA512_SCR(56); SHA512_SCR(57); SHA512_SCR(58); SHA512_SCR(59); + SHA512_SCR(60); SHA512_SCR(61); SHA512_SCR(62); SHA512_SCR(63); + SHA512_SCR(64); SHA512_SCR(65); SHA512_SCR(66); SHA512_SCR(67); + SHA512_SCR(68); SHA512_SCR(69); SHA512_SCR(70); SHA512_SCR(71); + SHA512_SCR(72); SHA512_SCR(73); SHA512_SCR(74); SHA512_SCR(75); + SHA512_SCR(76); SHA512_SCR(77); SHA512_SCR(78); SHA512_SCR(79); + + wv[0] = ctx->h[0]; wv[1] = ctx->h[1]; + wv[2] = ctx->h[2]; wv[3] = ctx->h[3]; + wv[4] = ctx->h[4]; wv[5] = ctx->h[5]; + wv[6] = ctx->h[6]; wv[7] = ctx->h[7]; + + j = 0; + + do { + SHA512_EXP(0,1,2,3,4,5,6,7,j); j++; + SHA512_EXP(7,0,1,2,3,4,5,6,j); j++; + SHA512_EXP(6,7,0,1,2,3,4,5,j); j++; + SHA512_EXP(5,6,7,0,1,2,3,4,j); j++; + SHA512_EXP(4,5,6,7,0,1,2,3,j); j++; + SHA512_EXP(3,4,5,6,7,0,1,2,j); j++; + SHA512_EXP(2,3,4,5,6,7,0,1,j); j++; + SHA512_EXP(1,2,3,4,5,6,7,0,j); j++; + } while (j < 80); + + ctx->h[0] += wv[0]; ctx->h[1] += wv[1]; + ctx->h[2] += wv[2]; ctx->h[3] += wv[3]; + ctx->h[4] += wv[4]; ctx->h[5] += wv[5]; + ctx->h[6] += wv[6]; ctx->h[7] += wv[7]; +#endif /* !UNROLL_LOOPS */ + } +} + +void sha512(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha512_ctx ctx; + + sha512_init(&ctx); + sha512_update(&ctx, message, len); + sha512_final(&ctx, digest); +} + +void sha512_init(sha512_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha512_h0[i]; + } +#else + ctx->h[0] = sha512_h0[0]; ctx->h[1] = sha512_h0[1]; + ctx->h[2] = sha512_h0[2]; ctx->h[3] = sha512_h0[3]; + ctx->h[4] = sha512_h0[4]; ctx->h[5] = sha512_h0[5]; + ctx->h[6] = sha512_h0[6]; ctx->h[7] = sha512_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha512_update(sha512_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA512_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA512_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA512_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha512_transf(ctx, ctx->block, 1); + sha512_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA512_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 7], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 7; +} + +void sha512_final(sha512_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = 1 + ((SHA512_BLOCK_SIZE - 17) + < (ctx->len % SHA512_BLOCK_SIZE)); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 7; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha512_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 8; i++) { + UNPACK64(ctx->h[i], &digest[i << 3]); + } +#else + UNPACK64(ctx->h[0], &digest[ 0]); + UNPACK64(ctx->h[1], &digest[ 8]); + UNPACK64(ctx->h[2], &digest[16]); + UNPACK64(ctx->h[3], &digest[24]); + UNPACK64(ctx->h[4], &digest[32]); + UNPACK64(ctx->h[5], &digest[40]); + UNPACK64(ctx->h[6], &digest[48]); + UNPACK64(ctx->h[7], &digest[56]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-384 functions */ + +void sha384(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha384_ctx ctx; + + sha384_init(&ctx); + sha384_update(&ctx, message, len); + sha384_final(&ctx, digest); +} + +void sha384_init(sha384_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha384_h0[i]; + } +#else + ctx->h[0] = sha384_h0[0]; ctx->h[1] = sha384_h0[1]; + ctx->h[2] = sha384_h0[2]; ctx->h[3] = sha384_h0[3]; + ctx->h[4] = sha384_h0[4]; ctx->h[5] = sha384_h0[5]; + ctx->h[6] = sha384_h0[6]; ctx->h[7] = sha384_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha384_update(sha384_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA384_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA384_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA384_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha512_transf(ctx, ctx->block, 1); + sha512_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA384_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 7], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 7; +} + +void sha384_final(sha384_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA384_BLOCK_SIZE - 17) + < (ctx->len % SHA384_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 7; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha512_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 6; i++) { + UNPACK64(ctx->h[i], &digest[i << 3]); + } +#else + UNPACK64(ctx->h[0], &digest[ 0]); + UNPACK64(ctx->h[1], &digest[ 8]); + UNPACK64(ctx->h[2], &digest[16]); + UNPACK64(ctx->h[3], &digest[24]); + UNPACK64(ctx->h[4], &digest[32]); + UNPACK64(ctx->h[5], &digest[40]); +#endif /* !UNROLL_LOOPS */ +} + +/* SHA-224 functions */ + +void sha224(const unsigned char *message, unsigned int len, + unsigned char *digest) +{ + sha224_ctx ctx; + + sha224_init(&ctx); + sha224_update(&ctx, message, len); + sha224_final(&ctx, digest); +} + +void sha224_init(sha224_ctx *ctx) +{ +#ifndef UNROLL_LOOPS + int i; + for (i = 0; i < 8; i++) { + ctx->h[i] = sha224_h0[i]; + } +#else + ctx->h[0] = sha224_h0[0]; ctx->h[1] = sha224_h0[1]; + ctx->h[2] = sha224_h0[2]; ctx->h[3] = sha224_h0[3]; + ctx->h[4] = sha224_h0[4]; ctx->h[5] = sha224_h0[5]; + ctx->h[6] = sha224_h0[6]; ctx->h[7] = sha224_h0[7]; +#endif /* !UNROLL_LOOPS */ + + ctx->len = 0; + ctx->tot_len = 0; +} + +void sha224_update(sha224_ctx *ctx, const unsigned char *message, + unsigned int len) +{ + unsigned int block_nb; + unsigned int new_len, rem_len, tmp_len; + const unsigned char *shifted_message; + + tmp_len = SHA224_BLOCK_SIZE - ctx->len; + rem_len = len < tmp_len ? len : tmp_len; + + memcpy(&ctx->block[ctx->len], message, rem_len); + + if (ctx->len + len < SHA224_BLOCK_SIZE) { + ctx->len += len; + return; + } + + new_len = len - rem_len; + block_nb = new_len / SHA224_BLOCK_SIZE; + + shifted_message = message + rem_len; + + sha256_transf(ctx, ctx->block, 1); + sha256_transf(ctx, shifted_message, block_nb); + + rem_len = new_len % SHA224_BLOCK_SIZE; + + memcpy(ctx->block, &shifted_message[block_nb << 6], + rem_len); + + ctx->len = rem_len; + ctx->tot_len += (block_nb + 1) << 6; +} + +void sha224_final(sha224_ctx *ctx, unsigned char *digest) +{ + unsigned int block_nb; + unsigned int pm_len; + unsigned int len_b; + +#ifndef UNROLL_LOOPS + int i; +#endif + + block_nb = (1 + ((SHA224_BLOCK_SIZE - 9) + < (ctx->len % SHA224_BLOCK_SIZE))); + + len_b = (ctx->tot_len + ctx->len) << 3; + pm_len = block_nb << 6; + + memset(ctx->block + ctx->len, 0, pm_len - ctx->len); + ctx->block[ctx->len] = 0x80; + UNPACK32(len_b, ctx->block + pm_len - 4); + + sha256_transf(ctx, ctx->block, block_nb); + +#ifndef UNROLL_LOOPS + for (i = 0 ; i < 7; i++) { + UNPACK32(ctx->h[i], &digest[i << 2]); + } +#else + UNPACK32(ctx->h[0], &digest[ 0]); + UNPACK32(ctx->h[1], &digest[ 4]); + UNPACK32(ctx->h[2], &digest[ 8]); + UNPACK32(ctx->h[3], &digest[12]); + UNPACK32(ctx->h[4], &digest[16]); + UNPACK32(ctx->h[5], &digest[20]); + UNPACK32(ctx->h[6], &digest[24]); +#endif /* !UNROLL_LOOPS */ +} + +#ifdef TEST_VECTORS + +/* FIPS 180-2 Validation tests */ + +#include +#include + +void test(const char *vector, unsigned char *digest, + unsigned int digest_size) +{ + char output[2 * SHA512_DIGEST_SIZE + 1]; + int i; + + output[2 * digest_size] = '\0'; + + for (i = 0; i < (int) digest_size ; i++) { + sprintf(output + 2 * i, "%02x", digest[i]); + } + + printf("H: %s\n", output); + if (strcmp(vector, output)) { + fprintf(stderr, "Test failed.\n"); + exit(EXIT_FAILURE); + } +} + +int main(void) +{ + static const char *vectors[4][3] = + { /* SHA-224 */ + { + "23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7", + "75388b16512776cc5dba5da1fd890150b0c6455cb4f58b1952522525", + "20794655980c91d8bbb4c1ea97618a4bf03f42581948b2ee4ee7ad67", + }, + /* SHA-256 */ + { + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1", + "cdc76e5c9914fb9281a1c7e284d73e67f1809a48a497200e046d39ccc7112cd0", + }, + /* SHA-384 */ + { + "cb00753f45a35e8bb5a03d699ac65007272c32ab0eded1631a8b605a43ff5bed" + "8086072ba1e7cc2358baeca134c825a7", + "09330c33f71147e83d192fc782cd1b4753111b173b3b05d22fa08086e3b0f712" + "fcc7c71a557e2db966c3e9fa91746039", + "9d0e1809716474cb086e834e310a4a1ced149e9c00f248527972cec5704c2a5b" + "07b8b3dc38ecc4ebae97ddd87f3d8985", + }, + /* SHA-512 */ + { + "ddaf35a193617abacc417349ae20413112e6fa4e89a97ea20a9eeee64b55d39a" + "2192992a274fc1a836ba3c23a3feebbd454d4423643ce80e2a9ac94fa54ca49f", + "8e959b75dae313da8cf4f72814fc143f8f7779c6eb9f7fa17299aeadb6889018" + "501d289e4900f7e4331b99dec4b5433ac7d329eeb6dd26545e96e55b874be909", + "e718483d0ce769644e2e42c7bc15b4638e1f98b13b2044285632a803afa973eb" + "de0ff244877ea60a4cb0432ce577c31beb009c5c2c49aa2e4eadb217ad8cc09b" + } + }; + + static const char message1[] = "abc"; + static const char message2a[] = "abcdbcdecdefdefgefghfghighijhi" + "jkijkljklmklmnlmnomnopnopq"; + static const char message2b[] = "abcdefghbcdefghicdefghijdefghijkefghij" + "klfghijklmghijklmnhijklmnoijklmnopjklm" + "nopqklmnopqrlmnopqrsmnopqrstnopqrstu"; + unsigned char *message3; + unsigned int message3_len = 1000000; + unsigned char digest[SHA512_DIGEST_SIZE]; + + message3 = malloc(message3_len); + if (message3 == NULL) { + fprintf(stderr, "Can't allocate memory\n"); + return -1; + } + memset(message3, 'a', message3_len); + + printf("SHA-2 FIPS 180-2 Validation tests\n\n"); + printf("SHA-224 Test vectors\n"); + + sha224((const unsigned char *) message1, strlen(message1), digest); + test(vectors[0][0], digest, SHA224_DIGEST_SIZE); + sha224((const unsigned char *) message2a, strlen(message2a), digest); + test(vectors[0][1], digest, SHA224_DIGEST_SIZE); + sha224(message3, message3_len, digest); + test(vectors[0][2], digest, SHA224_DIGEST_SIZE); + printf("\n"); + + printf("SHA-256 Test vectors\n"); + + sha256((const unsigned char *) message1, strlen(message1), digest); + test(vectors[1][0], digest, SHA256_DIGEST_SIZE); + sha256((const unsigned char *) message2a, strlen(message2a), digest); + test(vectors[1][1], digest, SHA256_DIGEST_SIZE); + sha256(message3, message3_len, digest); + test(vectors[1][2], digest, SHA256_DIGEST_SIZE); + printf("\n"); + + printf("SHA-384 Test vectors\n"); + + sha384((const unsigned char *) message1, strlen(message1), digest); + test(vectors[2][0], digest, SHA384_DIGEST_SIZE); + sha384((const unsigned char *)message2b, strlen(message2b), digest); + test(vectors[2][1], digest, SHA384_DIGEST_SIZE); + sha384(message3, message3_len, digest); + test(vectors[2][2], digest, SHA384_DIGEST_SIZE); + printf("\n"); + + printf("SHA-512 Test vectors\n"); + + sha512((const unsigned char *) message1, strlen(message1), digest); + test(vectors[3][0], digest, SHA512_DIGEST_SIZE); + sha512((const unsigned char *) message2b, strlen(message2b), digest); + test(vectors[3][1], digest, SHA512_DIGEST_SIZE); + sha512(message3, message3_len, digest); + test(vectors[3][2], digest, SHA512_DIGEST_SIZE); + printf("\n"); + + printf("All tests passed.\n"); + + return 0; +} + +#endif /* TEST_VECTORS */ + diff --git a/src/lib/hash/sha2/sha2.h b/src/lib/hash/sha2/sha2.h new file mode 100644 index 0000000..60f52e3 --- /dev/null +++ b/src/lib/hash/sha2/sha2.h @@ -0,0 +1,108 @@ +/* + * FIPS 180-2 SHA-224/256/384/512 implementation + * Last update: 02/02/2007 + * Issue date: 04/30/2005 + * + * Copyright (C) 2005, 2007 Olivier Gay + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the project nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef SHA2_H +#define SHA2_H + +#define SHA224_DIGEST_SIZE ( 224 / 8) +#define SHA256_DIGEST_SIZE ( 256 / 8) +#define SHA384_DIGEST_SIZE ( 384 / 8) +#define SHA512_DIGEST_SIZE ( 512 / 8) + +#define SHA256_BLOCK_SIZE ( 512 / 8) +#define SHA512_BLOCK_SIZE (1024 / 8) +#define SHA384_BLOCK_SIZE SHA512_BLOCK_SIZE +#define SHA224_BLOCK_SIZE SHA256_BLOCK_SIZE + +#ifndef SHA2_TYPES +#define SHA2_TYPES +typedef unsigned char uint8; +typedef unsigned int uint32; +typedef unsigned long long uint64; +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA256_BLOCK_SIZE]; + uint32 h[8]; +} sha256_ctx; + +typedef struct { + unsigned int tot_len; + unsigned int len; + unsigned char block[2 * SHA512_BLOCK_SIZE]; + uint64 h[8]; +} sha512_ctx; + +typedef sha512_ctx sha384_ctx; +typedef sha256_ctx sha224_ctx; + +void sha224_init(sha224_ctx *ctx); +void sha224_update(sha224_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha224_final(sha224_ctx *ctx, unsigned char *digest); +void sha224(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha256_init(sha256_ctx * ctx); +void sha256_update(sha256_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha256_final(sha256_ctx *ctx, unsigned char *digest); +void sha256(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha384_init(sha384_ctx *ctx); +void sha384_update(sha384_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha384_final(sha384_ctx *ctx, unsigned char *digest); +void sha384(const unsigned char *message, unsigned int len, + unsigned char *digest); + +void sha512_init(sha512_ctx *ctx); +void sha512_update(sha512_ctx *ctx, const unsigned char *message, + unsigned int len); +void sha512_final(sha512_ctx *ctx, unsigned char *digest); +void sha512(const unsigned char *message, unsigned int len, + unsigned char *digest); + +#ifdef __cplusplus +} +#endif + +#endif /* !SHA2_H */ + diff --git a/src/lib/header.c b/src/lib/header.c new file mode 100644 index 0000000..0d286c8 --- /dev/null +++ b/src/lib/header.c @@ -0,0 +1,140 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +int zck_read_initial(zckCtx *zck, int src_fd) { + char header[] = " "; + + if(!zck_read(src_fd, header, 6)) + return False; + + if(memcmp(header, "\0ZCK1", 5) != 0) { + zck_log(ZCK_LOG_ERROR, + "Invalid header, perhaps this is not a zck file?\n"); + return False; + } + + if(!zck_hash_setup(&(zck->hash_type), header[5])) + return False; + + return True; +} + +int zck_read_index_hash(zckCtx *zck, int src_fd) { + char *header; + header = zmalloc(zck->hash_type.digest_size); + if(!header) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->hash_type.digest_size); + return False; + } + if(!zck_read(src_fd, header, zck->hash_type.digest_size)) { + free(header); + return False; + } + zck->index_digest = header; + return True; +} + +int zck_read_comp_type(zckCtx *zck, int src_fd) { + int8_t comp_type; + + if(!zck_read(src_fd, (char *)&comp_type, 1)) + return False; + + if(!zck_set_compression_type(zck, comp_type)) + return False; + if(!zck_comp_init(zck)) + return False; + return True; +} + +int zck_read_index_size(zckCtx *zck, int src_fd) { + uint64_t index_size; + + if(!zck_read(src_fd, (char *)&index_size, sizeof(uint64_t))) + return False; + + zck->comp_index_size = le64toh(index_size); + return True; +} + +int zck_read_index(zckCtx *zck, int src_fd) { + char *index = zmalloc(zck->comp_index_size); + if(!index) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->comp_index_size); + return False; + } + if(!zck_read(src_fd, index, zck->comp_index_size)) { + free(index); + return False; + } + if(!zck_index_read(zck, index, zck->comp_index_size)) { + free(index); + return False; + } + free(index); + return True; +} + +int zck_read_header(zckCtx *zck, int src_fd) { + if(!zck_read_initial(zck, src_fd)) + return False; + if(!zck_read_index_hash(zck, src_fd)) + return False; + if(!zck_read_comp_type(zck, src_fd)) + return False; + if(!zck_read_index_size(zck, src_fd)) + return False; + if(!zck_read_index(zck, src_fd)) + return False; + return True; +} + +int zck_write_header(zckCtx *zck) { + uint64_t index_size; + + if(!zck_write(zck->fd, "\0ZCK1", 5)) + return False; + if(!zck_write(zck->fd, (const char *)&(zck->hash_type.type), 1)) + return False; + if(!zck_write(zck->fd, zck->index_digest, zck->hash_type.digest_size)) + return False; + if(!zck_write(zck->fd, (const char *)&(zck->comp.type), 1)) + return False; + index_size = htole64(zck->comp_index_size); + if(!zck_write(zck->fd, (const char *)&index_size, sizeof(uint64_t))) + return False; + return True; +} + diff --git a/src/lib/index/index_common.c b/src/lib/index/index_common.c new file mode 100644 index 0000000..9012228 --- /dev/null +++ b/src/lib/index/index_common.c @@ -0,0 +1,64 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +int zck_index_free(zckCtx *zck) { + if(zck->index.first) { + zckIndex *next; + zckIndex *tmp=zck->index.first; + while(tmp != NULL) { + next = tmp->next; + if(tmp->digest) + free(tmp->digest); + free(tmp); + tmp = next; + } + memset(&(zck->index), 0, sizeof(zckIndexInfo)); + } + if(zck->full_hash_digest) { + free(zck->full_hash_digest); + zck->full_hash_digest = NULL; + } + if(zck->full_hash.ctx) { + free(zck->full_hash.ctx); + zck->full_hash.ctx = NULL; + } + if(zck->comp_index) { + free(zck->comp_index); + zck->comp_index = NULL; + } + if(zck->index_digest) { + free(zck->index_digest); + zck->index_digest = NULL; + } + return True; +} diff --git a/src/lib/index/index_create.c b/src/lib/index/index_create.c new file mode 100644 index 0000000..3f8744e --- /dev/null +++ b/src/lib/index/index_create.c @@ -0,0 +1,207 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +int zck_index_finalize(zckCtx *zck) { + zckHash index_hash; + char *index; + char *index_loc; + size_t index_size = 0; + uint64_t index_count = 0; + + + zck->full_hash_digest = zck_hash_finalize(&(zck->full_hash)); + if(zck->full_hash_digest == NULL) + return False; + + index_size = 1; // Chunk hash type; + index_size += sizeof(uint64_t); // Number of index entries + index_size += zck->hash_type.digest_size; // Full hash digest + + /* Add digest size + 8 bytes for end location for each entry in index */ + if(zck->index.first) { + zckIndex *tmp = zck->index.first; + while(tmp) { + index_size += zck->index.hash_type->digest_size + sizeof(uint64_t); + tmp = tmp->next; + } + } + + /* Write index */ + index = zmalloc(index_size); + index_loc = index; + memcpy(index_loc, &(zck->index.hash_type->type), 1); + index_loc += 1; + index_count = htole64(zck->index.count); + memcpy(index_loc, &index_count, sizeof(uint64_t)); + index_loc += sizeof(uint64_t); + memcpy(index_loc, zck->full_hash_digest, zck->hash_type.digest_size); + index_loc += zck->hash_type.digest_size; + if(zck->index.first) { + zckIndex *tmp = zck->index.first; + while(tmp) { + uint64_t end = htole64(tmp->start + tmp->length); + memcpy(index_loc, tmp->digest, zck->index.hash_type->digest_size); + index_loc += zck->index.hash_type->digest_size; + memcpy(index_loc, &end, sizeof(uint64_t)); + index_loc += sizeof(uint64_t); + tmp = tmp->next; + } + } + + if(!zck->comp.compress(&zck->comp, index, index_size, &(zck->comp_index), + &(zck->comp_index_size), 0)) { + free(index); + return False; + } + index_size = htole64((uint64_t) zck->comp_index_size); + + /* Calculate hash of index, including compressed size at beginning */ + if(!zck_hash_init(&index_hash, &(zck->hash_type))) { + free(index); + return False; + } + if(!zck_hash_update(&index_hash, (const char *)&(zck->comp.type), 1)) { + free(index); + return False; + } + if(!zck_hash_update(&index_hash, (const char *)&index_size, + sizeof(uint64_t))) { + free(index); + return False; + } + if(!zck_hash_update(&index_hash, zck->comp_index, zck->comp_index_size)) { + free(index); + return False; + } + zck->index_digest = zck_hash_finalize(&index_hash); + if(zck->index_digest == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for index\n", + zck_hash_name_from_type(zck->hash_type.type)); + return False; + } + free(index); + return True; +} + +int zck_index_new_chunk(zckIndexInfo *index, char *digest, size_t length) { + if(index == NULL) { + zck_log(ZCK_LOG_ERROR, "Invalid index"); + return False; + } + zckIndex *idx = zmalloc(sizeof(zckIndex)); + if(idx == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckIndex)); + return False; + } + if(digest == NULL || length == 0) { + idx->digest = zmalloc(index->hash_type->digest_size); + if(idx->digest == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + index->hash_type->digest_size); + return False; + } + } else { + idx->digest = digest; + } + idx->start = index->length; + idx->length = length; + if(index->first == NULL) { + index->first = idx; + } else { + zckIndex *tmp=index->first; + while(tmp->next) + tmp = tmp->next; + tmp->next = idx; + } + index->count += 1; + index->length += length; + return True; +} + +int zck_index_add_dl_chunk(zckDL *dl, char *digest, size_t size) { + if(dl == NULL) { + zck_log(ZCK_LOG_ERROR, "Invalid dl context"); + return False; + } + zckIndex *new_index = zmalloc(sizeof(zckIndex)); + if(new_index == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckIndex)); + return False; + } + dl->index.count = dl->index.count + 1; +} + +int zck_index_add_chunk(zckCtx *zck, char *data, size_t size) { + zckIndex *new_index; + zckHash hash; + + if(zck == NULL) { + zck_log(ZCK_LOG_ERROR, "Invalid zck context"); + return False; + } + + if(size == 0) { + if(!zck_index_new_chunk(&(zck->index), NULL, size)) + return False; + } else { + if(!zck_hash_update(&(zck->full_hash), data, size)) { + free(new_index); + return False; + } + if(!zck_hash_init(&hash, zck->index.hash_type)) { + free(new_index); + return False; + } + if(!zck_hash_update(&hash, data, size)) { + free(new_index); + return False; + } + char *digest = zck_hash_finalize(&hash); + if(digest == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for new chunk\n", + zck_hash_name_from_type(zck->index.hash_type->type)); + return False; + } + if(!zck_index_new_chunk(&(zck->index), digest, size)) + return False; + } + return True; +} + +int zck_write_index(zckCtx *zck) { + return zck_write(zck->fd, zck->comp_index, zck->comp_index_size); +} diff --git a/src/lib/index/index_read.c b/src/lib/index/index_read.c new file mode 100644 index 0000000..ef2ea42 --- /dev/null +++ b/src/lib/index/index_read.c @@ -0,0 +1,146 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include + +#include "zck_private.h" + +int zck_index_read(zckCtx *zck, char *data, size_t size) { + zckHash index_hash; + char *digest; + uint64_t index_size; + uint64_t index_count; + uint8_t hash_type; + char *dst = NULL; + size_t dst_size = 0; + char *cur_loc; + + /* Check that index checksum matches stored checksum */ + zck_log(ZCK_LOG_DEBUG, "Reading index size\n"); + index_size = htole64(size); + if(!zck_hash_init(&index_hash, &(zck->hash_type))) + return False; + if(!zck_hash_update(&index_hash, (const char *)&(zck->comp.type), 1)) + return False; + if(!zck_hash_update(&index_hash, (const char *)&index_size, sizeof(uint64_t))) + return False; + if(!zck_hash_update(&index_hash, data, size)) + return False; + digest = zck_hash_finalize(&index_hash); + if(digest == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for header\n", + zck_hash_name_from_type(zck->hash_type.type)); + return False; + } + zck_log(ZCK_LOG_DEBUG, "Checking index checksum\n"); + if(memcmp(digest, zck->index_digest, zck->hash_type.digest_size) != 0) { + free(digest); + zck_log(ZCK_LOG_ERROR, "Index fails checksum test\n"); + return False; + } + free(digest); + zck_log(ZCK_LOG_DEBUG, "Decompressing index\n"); + if(!zck_decompress(zck, data, size, &dst, &dst_size)) { + zck_log(ZCK_LOG_ERROR, "Unable to decompress index\n"); + return False; + } + + /* Make sure there's at least enough data for full digest and index count */ + if(dst_size < zck->hash_type.digest_size + sizeof(uint64_t) + 1) { + zck_log(ZCK_LOG_ERROR, "Index is too small to read\n"); + if(dst) + free(dst); + return False; + } + + zckIndex *prev = zck->index.first; + zck->full_hash_digest = zmalloc(zck->hash_type.digest_size); + if(!zck->full_hash_digest) { + if(dst) + free(dst); + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->hash_type.digest_size); + return False; + } + memcpy(&hash_type, dst, 1); + if(!zck_hash_setup(zck->index.hash_type, hash_type)) { + if(dst) + free(dst); + return False; + } + + if((dst_size - (zck->hash_type.digest_size + sizeof(uint64_t)+ 1)) % + (zck->index.hash_type->digest_size + sizeof(uint64_t)) != 0) { + zck_log(ZCK_LOG_ERROR, "Index size is invalid\n"); + if(dst) + free(dst); + return False; + } + cur_loc = dst + 1; + memcpy(&index_count, cur_loc, sizeof(uint64_t)); + zck->index.count = le64toh(index_count); + cur_loc += sizeof(uint64_t); + memcpy(zck->full_hash_digest, cur_loc, zck->hash_type.digest_size); + cur_loc += zck->hash_type.digest_size; + uint64_t prev_loc = 0; + while(cur_loc < dst + dst_size) { + zckIndex *new = zmalloc(sizeof(zckIndex)); + if(!new) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + sizeof(zckIndex)); + return False; + } + uint64_t end = 0; + + new->digest = zmalloc(zck->index.hash_type->digest_size); + if(!new->digest) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + zck->index.hash_type->digest_size); + return False; + } + memcpy(new->digest, cur_loc, zck->index.hash_type->digest_size); + cur_loc += zck->index.hash_type->digest_size; + memcpy(&end, cur_loc, sizeof(uint64_t)); + new->start = prev_loc; + new->length = le64toh(end) - prev_loc; + prev_loc = le64toh(end); + zck->index.length += new->length; + cur_loc += sizeof(uint64_t); + if(prev) { + prev->next = new; + } else { + zck->index.first = new; + } + prev = new; + } + free(dst); + return True; +} diff --git a/src/lib/index/meson.build b/src/lib/index/meson.build new file mode 100644 index 0000000..6cc4999 --- /dev/null +++ b/src/lib/index/meson.build @@ -0,0 +1 @@ +sources += ['index/index_create.c', 'index/index_read.c', 'index/index_common.c'] diff --git a/src/lib/io.c b/src/lib/io.c new file mode 100644 index 0000000..cd0313d --- /dev/null +++ b/src/lib/io.c @@ -0,0 +1,74 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +#define BLK_SIZE 32768 + +int zck_read(int fd, char *data, size_t length) { + if(length == 0) + return True; + + if(read(fd, data, length) != length) { + zck_log(ZCK_LOG_ERROR, "Short read\n"); + return False; + } + + return True; +} + +int zck_write(int fd, const char *data, size_t length) { + if(length == 0) + return True; + + if(write(fd, data, length) != length) + return False; + return True; +} + +int zck_chunks_from_temp(zckCtx *zck) { + int read_count; + char *data = zmalloc(BLK_SIZE); + if(data == NULL) + return False; + + if(lseek(zck->temp_fd, 0, SEEK_SET) == -1) + return False; + + while((read_count = read(zck->temp_fd, data, BLK_SIZE)) > 0) { + if(read_count == -1 || !zck_write(zck->fd, data, read_count)) { + free(data); + return False; + } + } + free(data); + return True; +} diff --git a/src/lib/log.c b/src/lib/log.c new file mode 100644 index 0000000..8da7998 --- /dev/null +++ b/src/lib/log.c @@ -0,0 +1,47 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include + +#include "zck_private.h" + +static log_type log_level = ZCK_LOG_ERROR; + +void zck_set_log_level(log_type ll) { + log_level = ll; +} + +void zck_log(log_type lt, const char *format, ...) { + if(lt >= log_level) { + va_list args; + va_start(args, format); + vprintf(format, args); + va_end(args); + } +} diff --git a/src/lib/meson.build b/src/lib/meson.build new file mode 100644 index 0000000..0ec7488 --- /dev/null +++ b/src/lib/meson.build @@ -0,0 +1,13 @@ +sources = [] +subdir('comp') +subdir('hash') +subdir('index') +subdir('dl') +sources += ['zck.c', 'header.c', 'io.c', 'log.c'] +zcklib = shared_library('zck', + sources, + include_directories: inc, + dependencies: [zstd_dep, curl_dep], + install: true, + version: lib_version, + soversion: so_version) diff --git a/src/lib/zck.c b/src/lib/zck.c new file mode 100644 index 0000000..f04b9e5 --- /dev/null +++ b/src/lib/zck.c @@ -0,0 +1,338 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "zck_private.h" + +int zck_write_file(zckCtx *zck) { + zck_index_finalize(zck); + zck_log(ZCK_LOG_DEBUG, "Writing header\n"); + if(!zck_write_header(zck)) + return False; + zck_log(ZCK_LOG_DEBUG, "Writing index\n"); + if(!zck_write_index(zck)) + return False; + zck_log(ZCK_LOG_DEBUG, "Writing chunks\n"); + if(!zck_chunks_from_temp(zck)) + return False; + zck_log(ZCK_LOG_DEBUG, "Finished writing file, cleaning up\n"); + zck_index_free(zck); + zck_comp_close(zck); + if(zck->temp_fd) { + close(zck->temp_fd); + zck->temp_fd = 0; + } + + return True; +} + +void zck_free(zckCtx *zck) { + zck_index_free(zck); + zck_comp_close(zck); + zck_hash_close(&(zck->full_hash)); + zck_hash_close(&(zck->check_full_hash)); + if(zck->full_hash_digest) { + free(zck->full_hash_digest); + zck->full_hash_digest = NULL; + } + if(zck->index_digest) { + free(zck->index_digest); + zck->index_digest = NULL; + } + if(zck->temp_fd) { + close(zck->temp_fd); + zck->temp_fd = 0; + } + free(zck); + zck = NULL; +} + +zckCtx *zck_create() { + zckCtx *zck = zmalloc(sizeof(zckCtx)); + return zck; +} + +int zck_set_full_hash_type(zckCtx *zck, uint8_t hash_type) { + zck_log(ZCK_LOG_INFO, "Setting full hash to %s\n", + zck_hash_name_from_type(hash_type)); + if(!zck_hash_setup(&(zck->hash_type), hash_type)) { + zck_log(ZCK_LOG_ERROR, "Unable to set full hash to %s\n", + zck_hash_name_from_type(hash_type)); + return False; + } + zck_hash_close(&(zck->full_hash)); + if(!zck_hash_init(&(zck->full_hash), &(zck->hash_type))) { + zck_log(ZCK_LOG_ERROR, "Unable initialize full hash\n"); + return False; + } + return True; +} + +int zck_set_chunk_hash_type(zckCtx *zck, uint8_t hash_type) { + zck_log(ZCK_LOG_INFO, "Setting chunk hash to %s\n", + zck_hash_name_from_type(hash_type)); + if(!zck_hash_setup(zck->index.hash_type, hash_type)) { + zck_log(ZCK_LOG_ERROR, "Unable to set chunk hash to %s\n", + zck_hash_name_from_type(hash_type)); + return False; + } + return True; +} + +int zck_get_full_digest_size(zckCtx *zck) { + return zck->hash_type.digest_size; +} + +int zck_get_chunk_digest_size(zckCtx *zck) { + return zck->index.hash_type->digest_size; +} + +uint8_t zck_get_full_hash_type(zckCtx *zck) { + return zck->hash_type.type; +} + +uint8_t zck_get_chunk_hash_type(zckCtx *zck) { + return zck->index.hash_type->type; +} + +uint64_t zck_get_index_count(zckCtx *zck) { + return zck->index.count; +} + +zckIndexInfo *zck_get_index(zckCtx *zck) { + return &(zck->index); +} + +char *zck_get_index_digest(zckCtx *zck) { + return zck->index_digest; +} + +char *zck_get_full_digest(zckCtx *zck) { + return zck->full_hash_digest; +} + +int zck_get_tmp_fd() { + int temp_fd; + char *fname = NULL; + char template[] = "zcktempXXXXXX"; + char *tmpdir = getenv("TMPDIR"); + + if(tmpdir == NULL) { + tmpdir = "/tmp/"; + } + fname = zmalloc(strlen(template) + strlen(tmpdir) + 2); + if(fname == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", + strlen(template) + strlen(tmpdir) + 2); + return -1; + } + strncpy(fname, tmpdir, strlen(tmpdir)); + strncpy(fname+strlen(tmpdir), "/", 1); + strncpy(fname+strlen(tmpdir)+1, template, strlen(template)); + + temp_fd = mkstemp(fname); + if(temp_fd < 0) { + free(fname); + zck_log(ZCK_LOG_ERROR, "Unable to create temporary file\n"); + return -1; + } + /*if(unlink(fname) < 0) { + free(fname); + zck_log(ZCK_LOG_ERROR, "Unable to delete temporary file\n"); + return -1; + }*/ + free(fname); + return temp_fd; +} + +int zck_init_write (zckCtx *zck, int dst_fd) { + /* Clear zck */ + memset(zck, 0, sizeof(zckCtx)); + + zck->temp_fd = zck_get_tmp_fd(); + if(zck->temp_fd < 0) + return False; + + /* Set defaults */ +#ifdef ZCHUNK_ZSTD + zck_set_compression_type(zck, ZCK_COMP_ZSTD); +#else + zck_set_compression_type(zck, ZCK_COMP_NONE); +#endif + if(!zck_set_full_hash_type(zck, ZCK_HASH_SHA256)) + return False; + if(!zck_set_chunk_hash_type(zck, ZCK_HASH_SHA1)) + return False; + zck->fd = dst_fd; + + return True; +} + +int zck_import_dict(zckCtx *zck, char *data, size_t size) { + if(!zck_comp_close(zck)) + return False; + if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT, data)) + return False; + if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT_SIZE, &size)) + return False; + if(!zck_comp_init(zck)) + return False; + return True; +} + +int zck_validate_chunk(zckCtx *zck, char *data, size_t size, zckIndex *idx, + int chk_num) { + zckHash chunk_hash; + + /* Overall chunk checksum */ + if(!zck_hash_update(&(zck->check_full_hash), data, size)) { + zck_log(ZCK_LOG_ERROR, "Unable to update full file checksum\n"); + return False; + } + + /* Check chunk checksum */ + if(!zck_hash_init(&chunk_hash, zck->index.hash_type)) { + zck_log(ZCK_LOG_ERROR, + "Unable to initialize checksum for chunk %i\n", chk_num); + return False; + } + if(!zck_hash_update(&chunk_hash, data, size)) { + zck_log(ZCK_LOG_ERROR, + "Unable to update checksum for chunk %i\n", chk_num); + return False; + } + + char *digest = zck_hash_finalize(&chunk_hash); + if(!digest) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for chunk %i\n", + zck_hash_name_from_type(zck->index.hash_type->type), chk_num); + return False; + } + if(memcmp(digest, idx->digest, zck->index.hash_type->digest_size) != 0) { + free(digest); + zck_log(ZCK_LOG_ERROR, "Chunk %i failed %s checksum\n", + chk_num, zck_hash_name_from_type(zck->index.hash_type->type)); + return False; + } + free(digest); + return True; +} + +int zck_validate_file(zckCtx *zck) { + char *digest = zck_hash_finalize(&(zck->check_full_hash)); + if(digest == NULL) { + zck_log(ZCK_LOG_ERROR, + "Unable to calculate %s checksum for full file\n"); + return False; + } + if(memcmp(digest, zck->full_hash_digest, zck->hash_type.digest_size) != 0) { + free(digest); + zck_log(ZCK_LOG_ERROR, "Final file failed checksum\n"); + return False; + } + free(digest); + return True; +} + +int zck_decompress_to_file(zckCtx *zck, int src_fd, int dst_fd) { + if(!zck_read_header(zck, src_fd)) + return False; + if(!zck_hash_init(&(zck->check_full_hash), &(zck->hash_type))) + return False; + + int start = lseek(src_fd, 0, SEEK_CUR); + if(start == -1) { + zck_log(ZCK_LOG_ERROR, "Unable to set starting point source file: %s\n", + strerror(errno)); + return False; + } + + zckIndexInfo *index = zck_get_index(zck); + zckIndex *idx = index->first; + + /* Check if zck file is empty */ + for(int count=0; idx; count++) { + size_t csize = idx->length; + char *cdata; + + if(csize == 0) + continue; + + cdata = zmalloc(csize); + if(cdata == NULL) { + zck_log(ZCK_LOG_ERROR, "Unable to allocate %lu bytes\n", csize); + return False; + } + if(lseek(src_fd, start + idx->start, SEEK_SET) == -1) { + zck_log(ZCK_LOG_ERROR, "Unable to seek in source file: %s\n", + strerror(errno)); + return False; + } + if(!zck_read(src_fd, cdata, csize)) { + free(cdata); + zck_log(ZCK_LOG_ERROR, "Error reading chunk %i\n", count); + return False; + } + if(!zck_validate_chunk(zck, cdata, csize, idx, count)) { + free(cdata); + return False; + } + + size_t size = 0; + char *data = NULL; + if(!zck_decompress(zck, cdata, csize, &data, &size)) { + free(cdata); + zck_log(ZCK_LOG_ERROR, "Unable to decompress chunk %i\n", count); + return False; + } + free(cdata); + if(count == 0) { + if(!zck_import_dict(zck, data, size)) { + free(data); + return False; + } + } else { + if(!zck_write(dst_fd, data, size)) { + free(data); + zck_log(ZCK_LOG_ERROR, "Unable to write chunk %i\n", count); + return False; + } + } + free(data); + idx = idx->next; + } + if(!zck_validate_file(zck)) + return False; + return True; +} diff --git a/src/lib/zck_private.h b/src/lib/zck_private.h new file mode 100644 index 0000000..9dc965d --- /dev/null +++ b/src/lib/zck_private.h @@ -0,0 +1,122 @@ +#ifndef ZCK_PRIVATE_H +#define ZCK_PRIVATE_H +#include +#include +#include + +#define zmalloc(x) calloc(1, x) + +struct zckComp; + +typedef int (*finit)(struct zckComp *comp); +typedef int (*fparam)(struct zckComp *comp, int option, void *value); +typedef int (*fcomp)(struct zckComp *comp, const char *src, + const size_t src_size, char **dst, size_t *dst_size, + int use_dict); +typedef int (*fdecomp)(struct zckComp *comp, const char *src, + const size_t src_size, char **dst, size_t *dst_size, + int use_dict); +typedef int (*fcclose)(struct zckComp *comp); + +typedef enum log_type log_type; + +typedef struct zckHashType { + uint8_t type; + int digest_size; +} zckHashType; + +typedef struct { + zckHashType *type; + void *ctx; +} zckHash; + +/*typedef struct zckIndex zckIndex;*/ +typedef void CURL; + +typedef struct zckDL { + size_t dl; + size_t ul; + zckIndexInfo index; + CURL *curl_ctx; +} zckDL; + +typedef struct zckComp { + int started; + + uint8_t type; + int level; + + void *cctx; + void *dctx; + void *cdict_ctx; + void *ddict_ctx; + void *dict; + size_t dict_size; + finit init; + fparam set_parameter; + fcomp compress; + fdecomp decompress; + fcclose close; +} zckComp; + +typedef struct zckCtx { + int temp_fd; + int fd; + + char *full_hash_digest; + char *comp_index; + size_t comp_index_size; + zckIndexInfo index; + char *index_digest; + zckHash full_hash; + zckHash check_full_hash; + zckComp comp; + zckHashType hash_type; +} zckCtx; + +const char *zck_hash_name_from_type(uint8_t hash_type); +int zck_get_tmp_fd(); + +/* comp/comp.c */ +int zck_comp_init(zckCtx *zck); +int zck_compress(zckCtx *zck, const char *src, const size_t src_size); +int zck_decompress(zckCtx *zck, const char *src, const size_t src_size, char **dst, size_t *dst_size); +int zck_comp_close(zckCtx *zck); +int zck_set_compression_type(zckCtx *zck, uint8_t type); +int zck_set_comp_parameter(zckCtx *zck, int option, void *value); + +/* hash/hash.h */ +int zck_hash_setup(zckHashType *ht, int h); +int zck_hash_init(zckHash *hash, zckHashType *hash_type); +int zck_hash_update(zckHash *hash, const char *message, const size_t size); +char *zck_hash_finalize(zckHash *hash); +void zck_hash_close(zckHash *hash); + +/* index/index.c */ +int zck_index_read(zckCtx *zck, char *data, size_t size); +int zck_index_finalize(zckCtx *zck); +int zck_index_add_chunk(zckCtx *zck, char *data, size_t size); +int zck_index_free(zckCtx *zck); +int zck_write_index(zckCtx *zck); + +/* io.c */ +int zck_read(int fd, char *data, size_t length); +int zck_write(int fd, const char *data, size_t length); +int zck_chunks_from_temp(zckCtx *zck); + +/* header.c */ +int zck_read_initial(zckCtx *zck, int src_fd); +int zck_read_index_hash(zckCtx *zck, int src_fd); +int zck_read_comp_type(zckCtx *zck, int src_fd); +int zck_read_index_size(zckCtx *zck, int src_fd); +int zck_read_index(zckCtx *zck, int src_fd); +int zck_read_header(zckCtx *zck, int src_fd); +int zck_write_header(zckCtx *zck); + +/* dl/range.c */ +char *zck_range_get_char(zckRange **range, int max_ranges); +int zck_range_add(zckRangeInfo *info, uint64_t start, uint64_t end); + +/* log.c */ +void zck_log(log_type lt, const char *format, ...); +#endif diff --git a/src/memmem.c b/src/memmem.c new file mode 100644 index 0000000..e72501b --- /dev/null +++ b/src/memmem.c @@ -0,0 +1,64 @@ +/* + * Copyright (C) 2008 The Android Open Source Project + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in + * the documentation and/or other materials provided with the + * distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS + * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT + * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/* + * This uses the "Not So Naive" algorithm, a very simple but + * usually effective algorithm, see: + * http://www-igm.univ-mlv.fr/~lecroq/string/ + */ +#include + +void *memmem(const void *haystack, size_t n, const void *needle, size_t m) +{ + if (m > n || !m || !n) + return NULL; + + if (__builtin_expect((m > 1), 1)) { + const unsigned char* y = (const unsigned char*) haystack; + const unsigned char* x = (const unsigned char*) needle; + size_t j = 0; + size_t k = 1, l = 2; + + if (x[0] == x[1]) { + k = 2; + l = 1; + } + while (j <= n-m) { + if (x[1] != y[j+1]) { + j += k; + } else { + if (!memcmp(x+2, y+j+2, m-2) && x[0] == y[j]) + return (void*) &y[j]; + j += l; + } + } + } else { + /* degenerate case */ + return memchr(haystack, ((unsigned char*)needle)[0], n); + } + return NULL; +} diff --git a/src/memmem.h b/src/memmem.h new file mode 100644 index 0000000..2eb0206 --- /dev/null +++ b/src/memmem.h @@ -0,0 +1,4 @@ +#ifndef MEMMEM_H +#define MEMMEM_H +void *memmem(const void *haystack, size_t n, const void *needle, size_t m); +#endif diff --git a/src/meson.build b/src/meson.build new file mode 100644 index 0000000..d4364f6 --- /dev/null +++ b/src/meson.build @@ -0,0 +1,6 @@ +subdir('lib') +executable('zck', ['zck.c'], include_directories: inc, link_with: zcklib) +executable('unzck', ['unzck.c'], include_directories: inc, link_with: zcklib) +executable('zckdl', ['zck_dl.c'], include_directories: inc, link_with: zcklib) +executable('zck_read_header', ['zck_read_header.c'], include_directories: inc, link_with: zcklib) +executable('zck_delta_size', ['zck_delta_size.c', 'memmem.c'], include_directories: inc, link_with: zcklib) diff --git a/src/unzck.c b/src/unzck.c new file mode 100644 index 0000000..bffce2f --- /dev/null +++ b/src/unzck.c @@ -0,0 +1,80 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main (int argc, char *argv[]) { + zckCtx *zck = zck_create(); + char *out_name; + if(zck == NULL) + exit(1); + + zck_set_log_level(ZCK_LOG_DEBUG); + + if(argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(1); + } + + int src_fd = open(argv[1], O_RDONLY); + if(src_fd < 0) { + printf("Unable to open %s\n", argv[1]); + perror(""); + exit(1); + } + + out_name = malloc(strlen(argv[1]) - 3); + snprintf(out_name, strlen(argv[1]) - 3, "%s", argv[1]); + + int dst_fd = open(out_name, O_EXCL | O_WRONLY | O_CREAT, 0644); + if(dst_fd < 0) { + printf("Unable to open %s", out_name); + perror(""); + free(out_name); + exit(1); + } + + + if(!zck_decompress_to_file(zck, src_fd, dst_fd)) { + unlink(out_name); + free(out_name); + close(src_fd); + close(dst_fd); + zck_free(zck); + } + free(out_name); + close(src_fd); + close(dst_fd); + zck_free(zck); +} diff --git a/src/zck.c b/src/zck.c new file mode 100644 index 0000000..d995d0c --- /dev/null +++ b/src/zck.c @@ -0,0 +1,212 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "memmem.h" + +#define WINDOW_SIZE 4096 +#define MATCH_SUM WINDOW_SIZE-1 + +int main (int argc, char *argv[]) { + char *out_name; + char *dict = NULL; + size_t dict_size = 0; + zckCtx *zck = zck_create(); + + zck_set_log_level(ZCK_LOG_DEBUG); + + if(argc < 3 || argc > 4) { + printf("Usage: %s [optional dictionary]\n", argv[0]); + exit(1); + } + out_name = malloc(strlen(argv[1]) + 5); + snprintf(out_name, strlen(argv[1]) + 5, "%s.zck", argv[1]); + if(argc == 4) { + int dict_fd = open(argv[3], O_RDONLY); + if(dict_fd < 0) { + printf("Unable to open dictionary %s for reading", argv[3]); + perror(""); + exit(1); + } + dict_size = lseek(dict_fd, 0, SEEK_END); + if(dict_size < 0) { + perror("Unable to seek to end of dictionary"); + exit(1); + } + if(lseek(dict_fd, 0, SEEK_SET) < 0) { + perror("Unable to seek to beginning of dictionary"); + exit(1); + } + dict = malloc(dict_size); + read(dict_fd, dict, dict_size); + close(dict_fd); + } + + int dst_fd = open(out_name, O_EXCL | O_WRONLY | O_CREAT, 0644); + if(dst_fd < 0) { + printf("Unable to open %s", out_name); + perror(""); + if(dict) { + free(dict); + dict = NULL; + } + free(out_name); + exit(1); + } + free(out_name); + if(!zck_init_write(zck, dst_fd)) + exit(1); + + /*if(!zck_set_compression_type(zck, ZCK_COMP_NONE)) { + perror("Unable to set compression type\n"); + exit(1); + }*/ + if(dict_size > 0) { + if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT, dict)) + exit(1); + if(!zck_set_comp_parameter(zck, ZCK_COMMON_DICT_SIZE, &dict_size)) + exit(1); + } + if(!zck_comp_init(zck)) + exit(1); + free(dict); + + char *data; + int in_fd = open(argv[1], O_RDONLY); + int in_size = 0; + if(in_fd < 0) { + printf("Unable to open %s for reading", argv[1]); + perror(""); + exit(1); + } + in_size = lseek(in_fd, 0, SEEK_END); + if(in_size < 0) { + perror("Unable to seek to end of input file"); + exit(1); + } + if(lseek(in_fd, 0, SEEK_SET) < 0) { + perror("Unable to seek to beginning of input file"); + exit(1); + } + if(in_size > 0) { + data = malloc(in_size); + read(in_fd, data, in_size); + close(in_fd); + + if(True) { + char *found = data; + char *search = found; + char *prev_srpm = memmem(search, in_size - (search-data), " next) + prev_srpm = NULL; + if(prev_srpm) { + int matched=0; + char prev = '\0'; + for(int i=0;;i++) { + if(next_srpm[i] != prev_srpm[i]) + break; + if(next_srpm[i] == '/' && prev == '<') { + matched = 1; + break; + } + prev = next_srpm[i]; + } + if(matched) { + search = next + 1; + if(search > data + in_size) + search = data + in_size; + continue; + } + + } + prev_srpm = next_srpm; + printf("Compressing %li bytes\n", next-found); + if(!zck_compress(zck, found, next-found)) + exit(1); + found = next; + search = next + 1; + if(search > data + in_size) + search = data + in_size; + } else { + printf("Completing %li bytes\n", data+in_size-found); + if(!zck_compress(zck, found, data+in_size-found)) + exit(1); + search = NULL; + } + } + } else { + char *cur_loc = data; + char *start = data; + char *window_loc; + int window_sum; + + while(cur_loc < data + in_size) { + window_sum = 0; + window_loc = cur_loc; + if(cur_loc + WINDOW_SIZE < data + in_size) { + for(int i=0; i + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main (int argc, char *argv[]) { + zckCtx *zck_src = zck_create(); + zckCtx *zck_tgt = zck_create(); + + /*zck_set_log_level(ZCK_LOG_DEBUG);*/ + + if(argc != 3) { + printf("Usage: %s \n", argv[0]); + exit(1); + } + + int src_fd = open(argv[1], O_RDONLY); + if(src_fd < 0) { + printf("Unable to open %s\n", argv[1]); + perror(""); + exit(1); + } + if(!zck_read_header(zck_src, src_fd)) { + printf("Unable to open %s\n", argv[1]); + exit(1); + } + close(src_fd); + + int tgt_fd = open(argv[2], O_RDONLY); + if(tgt_fd < 0) { + printf("Unable to open %s\n", argv[2]); + perror(""); + exit(1); + } + if(!zck_read_header(zck_tgt, tgt_fd)) { + printf("Unable to open %s\n", argv[2]); + exit(1); + } + close(tgt_fd); + + if(zck_get_chunk_hash_type(zck_tgt) != zck_get_chunk_hash_type(zck_src)) { + printf("ERROR: Chunk hash types don't match:\n"); + printf(" %s: %s\n", argv[1], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_tgt))); + printf(" %s: %s\n", argv[2], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_src))); + return 1; + } + zckIndex *tgt_idx = zck_get_index(zck_tgt); + zckIndex *src_idx = zck_get_index(zck_src); + if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) != 0) + printf("WARNING: Dicts don't match\n"); + int dl_size = 0; + int total_size = 0; + int matched_chunks = 0; + while(tgt_idx) { + int found = False; + src_idx = zck_get_index(zck_src); + + while(src_idx) { + if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) == 0) { + found = True; + break; + } + src_idx = src_idx->next; + } + if(!found) { + dl_size += tgt_idx->length; + } else { + matched_chunks += 1; + } + total_size += tgt_idx->length; + tgt_idx = tgt_idx->next; + } + printf("Would download %i of %i bytes\n", dl_size, total_size); + printf("Matched %i of %lu chunks\n", matched_chunks, zck_get_index_count(zck_tgt)); + zck_free(zck_tgt); + zck_free(zck_src); +} diff --git a/src/zck_dl.c b/src/zck_dl.c new file mode 100644 index 0000000..22ed8bf --- /dev/null +++ b/src/zck_dl.c @@ -0,0 +1,146 @@ +/* + * Copyright 2018 Jonathan Dieter + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int main (int argc, char *argv[]) { + zckCtx *zck_src = zck_create(); + zckCtx *zck_tgt = zck_create(); + + if(zck_src == NULL || zck_tgt == NULL) + exit(1); + + zck_dl_global_init(); + zck_set_log_level(ZCK_LOG_DEBUG); + + if(argc != 3) { + printf("Usage: %s \n", argv[0]); + exit(1); + } + + int src_fd = open(argv[1], O_RDONLY); + if(src_fd < 0) { + printf("Unable to open %s\n", argv[1]); + perror(""); + exit(1); + } + if(!zck_read_header(zck_src, src_fd)) { + printf("Unable to open %s\n", argv[1]); + exit(1); + } + close(src_fd); + + zckDL *dl = zck_dl_init(); + if(dl == NULL) + exit(1); + if(!zck_dl_get_header(zck_tgt, dl, argv[2])) + exit(1); + + zckRangeInfo info = {0}; + if(!zck_range_get_need_dl(&info, zck_src, zck_tgt)) + exit(1); + + + printf("Downloaded %lu bytes\n", zck_dl_get_bytes_downloaded(dl)); + zck_range_close(&info); + zck_dl_free(dl); + zck_free(zck_tgt); + zck_free(zck_src); + zck_dl_global_cleanup(); + exit(0); + + /* + if(zck_get_chunk_hash_type(zck_tgt) != zck_get_chunk_hash_type(zck_src)) { + printf("ERROR: Chunk hash types don't match:\n"); + printf(" %s: %s\n", argv[1], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_tgt))); + printf(" %s: %s\n", argv[2], zck_hash_name_from_type(zck_get_chunk_hash_type(zck_src))); + return 1; + } + zckIndex *tgt_idx = zck_get_index(zck_tgt); + zckIndex *src_idx = zck_get_index(zck_src); + if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) != 0) + printf("WARNING: Dicts don't match\n"); + int dl_size = 0; + int total_size = 0; + int matched_chunks = 0; + while(tgt_idx) { + int found = False; + src_idx = zck_get_index(zck_src); + + while(src_idx) { + if(memcmp(tgt_idx->digest, src_idx->digest, zck_get_chunk_digest_size(zck_tgt)) == 0) { + found = True; + break; + } + src_idx = src_idx->next; + } + if(!found) { + dl_size += tgt_idx->length; + } else { + matched_chunks += 1; + } + total_size += tgt_idx->length; + tgt_idx = tgt_idx->next; + } + printf("Would download %i of %i bytes\n", dl_size, total_size); + printf("Matched %i of %lu chunks\n", matched_chunks, zck_get_index_count(zck_tgt)); + + zckRangeInfo info = {0}; + if(!zck_range_get_need_dl(&info, zck_src, zck_tgt)) + exit(1); + zckRange *tmp = info.first; + while(tmp) { + printf("Range: %lu - %lu\n", tmp->start, tmp->end); + tmp = tmp->next; + } + printf("Count: %u\n", info.count); + zck_range_calc_segments(&info, 5); + char **ra = calloc(sizeof(char*), info.segments); + if(!ra) { + printf("Unable to allocate %lu bytes\n", sizeof(char*) * info.segments); + exit(1); + } + if(!zck_range_get_array(&info, ra)) + exit(1); + for(int i=0; i + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +int main (int argc, char *argv[]) { + zckCtx *zck = zck_create(); + + zck_set_log_level(ZCK_LOG_DEBUG); + + if(argc != 2) { + printf("Usage: %s \n", argv[0]); + exit(1); + } + + int src_fd = open(argv[1], O_RDONLY); + if(src_fd < 0) { + printf("Unable to open %s\n", argv[1]); + perror(""); + exit(1); + } + if(!zck_read_header(zck, src_fd)) { + perror("Unable to read header\n"); + exit(1); + } + close(src_fd); + zckIndex *idx = zck_get_index(zck); + printf("Overall checksum type: %s\n", zck_hash_name_from_type(zck_get_full_hash_type(zck))); + printf("Index checksum: "); + char *digest = zck_get_index_digest(zck); + for(int i=0; idigest[i]); + printf(" %12lu %12lu\n", idx->start, idx->length); + idx = idx->next; + } + + zck_free(zck); +} -- 2.30.2